thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 16:50:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:yandex/ClickHouse

Browse Source
This commit is contained in:
Alexey Milovidov 2018-08-20 18:50:09 +03:00
commit 465ff07398
286 changed files with 6033 additions and 1753 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
CHANGELOG.draft.md
View File

@ -1 +1,31 @@
## RU
## ClickHouse release 18.10.3, 2018-08-13
### Новые возможности:
* поддержка межсерверной репликации по HTTPS
* MurmurHash
* ODBCDriver2 с поддержкой NULL-ов
* поддержка UUID в ключевых колонках (экспериментально)
### Улучшения:
* добавлена поддержка SETTINGS для движка Kafka
* поддежка пустых кусков после мержей в движках Summing, Collapsing and VersionedCollapsing
* удаление старых записей о полностью выполнившихся мутациях
* исправлена логика REPLACE PARTITION для движка RplicatedMergeTree
* добавлена системная таблица system.merge_tree_settings
* в системную таблицу system.tables добавлены столбцы зависимостей: dependencies_database и dependencies_table
* заменен аллокатор, теперь используется jemalloc вместо tcmalloc
* улучшена валидация connection string ODBC
* удалена поддержка CHECK TABLE для распределенных таблиц
* добавлены stateful тесты (пока без данных)
* добавлена опция конфига max_partition_size_to_drop
* добавлена настройка output_format_json_escape_slashes
* добавлена настройка max_fetch_partition_retries_count
* добавлена настройка prefer_localhost_replica
* добавлены libressl, unixodbc и mariadb-connector-c как сабмодули
### Исправление ошибок:
* #2786
* #2777
* #2795

4
CMakeLists.txt
View File

@ -1,5 +1,5 @@
project (ClickHouse)
cmake_minimum_required (VERSION 2.8)
cmake_minimum_required (VERSION 3.3)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${ClickHouse_SOURCE_DIR}/cmake/Modules/")
@ -218,7 +218,7 @@ else ()
set (CLICKHOUSE_ETC_DIR "${CMAKE_INSTALL_PREFIX}/etc")
endif ()
option (UNBUNDLED "Try find all libraries in system (if fail - use bundled from contrib/)" OFF)
option (UNBUNDLED "Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages." OFF)
if (UNBUNDLED)
set(NOT_UNBUNDLED 0)
else ()

8
cmake/arch.cmake
View File

@ -7,9 +7,9 @@ endif ()
if (CMAKE_LIBRARY_ARCHITECTURE MATCHES "i386")
set (ARCH_I386 1)
endif ()
if ( ( ARCH_ARM AND NOT ARCH_AARCH64 ) OR ARCH_I386)
if ((ARCH_ARM AND NOT ARCH_AARCH64) OR ARCH_I386)
set (ARCH_32 1)
message (WARNING "Support for 32bit platforms is highly experimental")
message (FATAL_ERROR "32bit platforms are not supported")
endif ()
if (CMAKE_SYSTEM MATCHES "Linux")
@ -24,7 +24,3 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set (COMPILER_CLANG 1)
endif ()
if (OS_LINUX AND CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
set (OS_LINUX_X86_64 1)
endif ()

2
cmake/find_rdkafka.cmake
View File

@ -2,7 +2,7 @@ option (ENABLE_RDKAFKA "Enable kafka" ON)
if (ENABLE_RDKAFKA)
if (OS_LINUX_X86_64)
if (OS_LINUX)
option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ${NOT_UNBUNDLED})
endif ()

2
cmake/find_zlib.cmake
View File

@ -1,4 +1,4 @@
if (NOT OS_FREEBSD)
if (NOT OS_FREEBSD AND NOT APPLE)
option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED})
endif ()

4
contrib/CMakeLists.txt vendored
View File

@ -1,10 +1,10 @@
# Third-party libraries may have substandard code.
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-stringop-overflow")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-implicit-fallthrough -std=c++1z")
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format -Wno-parentheses-equality")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-format -std=c++1z")
endif ()

12
dbms/cmake/version.cmake
View File

@ -1,11 +1,11 @@
# This strings autochanged from release_lib.sh:
set(VERSION_REVISION 54405 CACHE STRING "")
set(VERSION_REVISION 54406 CACHE STRING "")
set(VERSION_MAJOR 18 CACHE STRING "")
set(VERSION_MINOR 10 CACHE STRING "")
set(VERSION_PATCH 3 CACHE STRING "")
set(VERSION_GITHASH 1fa1b34f1ab01ea2e1a833eebd36a4806e529f52 CACHE STRING "")
set(VERSION_DESCRIBE v18.10.3-testing CACHE STRING "")
set(VERSION_STRING 18.10.3 CACHE STRING "")
set(VERSION_MINOR 11 CACHE STRING "")
set(VERSION_PATCH 0 CACHE STRING "")
set(VERSION_GITHASH 76af46ed5d223b3a7af92e31eae291174da16355 CACHE STRING "")
set(VERSION_DESCRIBE v18.11.0-testing CACHE STRING "")
set(VERSION_STRING 18.11.0 CACHE STRING "")
# end of autochange
set(VERSION_EXTRA "" CACHE STRING "")

331
dbms/programs/client/Client.cpp
View File

@ -1,3 +1,5 @@
#include "TestHint.h"
#include <port/unistd.h>
#include <stdlib.h>
#include <fcntl.h>
@ -20,7 +22,6 @@
#include <Common/Stopwatch.h>
#include <Common/Exception.h>
#include <Common/ShellCommand.h>
#include <Common/ExternalTable.h>
#include <Common/UnicodeBar.h>
#include <Common/formatReadable.h>
#include <Common/NetException.h>
@ -31,6 +32,7 @@
#include <Common/config_version.h>
#include <Core/Types.h>
#include <Core/QueryProcessingStage.h>
#include <Core/ExternalTable.h>
#include <IO/ReadBufferFromFileDescriptor.h>
#include <IO/WriteBufferFromFileDescriptor.h>
#include <IO/WriteBufferFromFile.h>
@ -39,6 +41,7 @@
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <DataStreams/AsynchronousBlockInputStream.h>
#include <DataStreams/InternalTextLogsRowOutputStream.h>
#include <Parsers/ParserQuery.h>
#include <Parsers/ASTSetQuery.h>
#include <Parsers/ASTUseQuery.h>
@ -90,102 +93,6 @@ namespace ErrorCodes
}
/// Checks expected server and client error codes in testmode.
/// To enable it add special comment after the query: "-- { serverError 60 }" or "-- { clientError 20 }".
class TestHint
{
public:
TestHint(bool enabled_, const String & query)
: enabled(enabled_),
server_error(0),
client_error(0)
{
if (!enabled_)
return;
size_t pos = query.find("--");
if (pos != String::npos && query.find("--", pos + 2) != String::npos)
return; /// It's not last comment. Hint belongs to commented query.
if (pos != String::npos)
{
pos = query.find('{', pos + 2);
if (pos != String::npos)
{
String hint = query.substr(pos + 1);
pos = hint.find('}');
hint.resize(pos);
parse(hint);
}
}
}
/// @returns true if it's possible to continue without reconnect
bool checkActual(int & actual_server_error, int & actual_client_error,
bool & got_exception, std::unique_ptr<Exception> & last_exception) const
{
if (!enabled)
return true;
if (allErrorsExpected(actual_server_error, actual_client_error))
{
got_exception = false;
last_exception.reset();
actual_server_error = 0;
actual_client_error = 0;
return false;
}
if (lostExpectedError(actual_server_error, actual_client_error))
{
std::cerr << "Success when error expected. It expects server error "
<< server_error << ", client error " << client_error << "." << std::endl;
got_exception = true;
last_exception = std::make_unique<Exception>("Success when error expected", ErrorCodes::LOGICAL_ERROR); /// return error to OS
return false;
}
return true;
}
int serverError() const { return server_error; }
int clientError() const { return client_error; }
private:
bool enabled;
int server_error;
int client_error;
void parse(const String & hint)
{
std::stringstream ss;
ss << hint;
while (!ss.eof())
{
String item;
ss >> item;
if (item.empty())
break;
if (item == "serverError")
ss >> server_error;
else if (item == "clientError")
ss >> client_error;
}
}
bool allErrorsExpected(int actual_server_error, int actual_client_error) const
{
return (server_error || client_error) && (server_error == actual_server_error) && (client_error == actual_client_error);
}
bool lostExpectedError(int actual_server_error, int actual_client_error) const
{
return (server_error && !actual_server_error) || (client_error && !actual_client_error);
}
};
class Client : public Poco::Util::Application
{
public:
@ -235,6 +142,11 @@ private:
std::optional<WriteBufferFromFile> out_file_buf;
BlockOutputStreamPtr block_out_stream;
/// The user could specify special file for server logs (stderr by default)
std::unique_ptr<WriteBuffer> out_logs_buf;
String server_logs_file;
BlockOutputStreamPtr logs_out_stream;
String home_path;
String current_profile;
@ -408,20 +320,10 @@ private:
/// If exception code isn't zero, we should return non-zero return code anyway.
return e.code() ? e.code() : -1;
}
catch (const Poco::Exception & e)
{
std::cerr << "Poco::Exception: " << e.displayText() << std::endl;
return ErrorCodes::POCO_EXCEPTION;
}
catch (const std::exception & e)
{
std::cerr << "std::exception: " << e.what() << std::endl;
return ErrorCodes::STD_EXCEPTION;
}
catch (...)
{
std::cerr << "Unknown exception" << std::endl;
return ErrorCodes::UNKNOWN_EXCEPTION;
std::cerr << getCurrentExceptionMessage(false) << std::endl;
return getCurrentExceptionCode();
}
}
@ -469,7 +371,12 @@ private:
format_max_block_size = config().getInt("format_max_block_size", context.getSettingsRef().max_block_size);
insert_format = "Values";
insert_format_max_block_size = config().getInt("insert_format_max_block_size", context.getSettingsRef().max_insert_block_size);
/// Setting value from cmd arg overrides one from config
if (context.getSettingsRef().max_insert_block_size.changed)
insert_format_max_block_size = context.getSettingsRef().max_insert_block_size;
else
insert_format_max_block_size = config().getInt("insert_format_max_block_size", context.getSettingsRef().max_insert_block_size);
if (!is_interactive)
{
@ -782,6 +689,7 @@ private:
{
const char * pos = begin;
ASTPtr ast = parseQuery(pos, end, true);
if (!ast)
{
if (ignore_error)
@ -797,7 +705,7 @@ private:
return true;
}
ASTInsertQuery * insert = typeid_cast<ASTInsertQuery *>(&*ast);
ASTInsertQuery * insert = typeid_cast<ASTInsertQuery *>(ast.get());
if (insert && insert->data)
{
@ -990,7 +898,7 @@ private:
/// If structure was received (thus, server has not thrown an exception),
/// send our data with that structure.
sendData(sample);
receivePacket();
receiveEndOfQuery();
}
}
@ -1072,6 +980,11 @@ private:
connection->sendData(block);
processed_rows += block.rows();
/// Check if server send Log packet
auto packet_type = connection->checkPacket();
if (packet_type && *packet_type == Protocol::Server::Log)
receiveAndProcessPacket();
if (!block)
break;
}
@ -1083,18 +996,28 @@ private:
/// Flush all buffers.
void resetOutput()
{
block_out_stream = nullptr;
block_out_stream.reset();
logs_out_stream.reset();
if (pager_cmd)
{
pager_cmd->in.close();
pager_cmd->wait();
}
pager_cmd = nullptr;
if (out_file_buf)
{
out_file_buf->next();
out_file_buf.reset();
}
if (out_logs_buf)
{
out_logs_buf->next();
out_logs_buf.reset();
}
std_out.next();
}
@ -1127,7 +1050,7 @@ private:
continue; /// If there is no new data, continue checking whether the query was cancelled after a timeout.
}
if (!receivePacket())
if (!receiveAndProcessPacket())
break;
}
@ -1138,7 +1061,7 @@ private:
/// Receive a part of the result, or progress info or an exception and process it.
/// Returns true if one should continue receiving packets.
bool receivePacket()
bool receiveAndProcessPacket()
{
Connection::Packet packet = connection->receivePacket();
@ -1169,6 +1092,10 @@ private:
last_exception = std::move(packet.exception);
return false;
case Protocol::Server::Log:
onLogData(packet.block);
return true;
case Protocol::Server::EndOfStream:
onEndOfStream();
return false;
@ -1182,22 +1109,59 @@ private:
/// Receive the block that serves as an example of the structure of table where data will be inserted.
bool receiveSampleBlock(Block & out)
{
Connection::Packet packet = connection->receivePacket();
switch (packet.type)
while (true)
{
case Protocol::Server::Data:
out = packet.block;
return true;
Connection::Packet packet = connection->receivePacket();
case Protocol::Server::Exception:
onException(*packet.exception);
last_exception = std::move(packet.exception);
return false;
switch (packet.type)
{
case Protocol::Server::Data:
out = packet.block;
return true;
default:
throw NetException("Unexpected packet from server (expected Data, got "
+ String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER);
case Protocol::Server::Exception:
onException(*packet.exception);
last_exception = std::move(packet.exception);
return false;
case Protocol::Server::Log:
onLogData(packet.block);
break;
default:
throw NetException("Unexpected packet from server (expected Data, Exception or Log, got "
+ String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER);
}
}
}
/// Process Log packets, exit when recieve Exception or EndOfStream
bool receiveEndOfQuery()
{
while (true)
{
Connection::Packet packet = connection->receivePacket();
switch (packet.type)
{
case Protocol::Server::EndOfStream:
onEndOfStream();
return true;
case Protocol::Server::Exception:
onException(*packet.exception);
last_exception = std::move(packet.exception);
return false;
case Protocol::Server::Log:
onLogData(packet.block);
break;
default:
throw NetException("Unexpected packet from server (expected Exception, EndOfStream or Log, got "
+ String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER);
}
}
}
@ -1253,6 +1217,38 @@ private:
}
void initLogsOutputStream()
{
if (!logs_out_stream)
{
WriteBuffer * wb = out_logs_buf.get();
if (!out_logs_buf)
{
if (server_logs_file.empty())
{
/// Use stderr by default
out_logs_buf = std::make_unique<WriteBufferFromFileDescriptor>(STDERR_FILENO);
wb = out_logs_buf.get();
}
else if (server_logs_file == "-")
{
/// Use stdout if --server_logs_file=- specified
wb = &std_out;
}
else
{
out_logs_buf = std::make_unique<WriteBufferFromFile>(server_logs_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_APPEND | O_CREAT);
wb = out_logs_buf.get();
}
}
logs_out_stream = std::make_shared<InternalTextLogsRowOutputStream>(*wb);
logs_out_stream->writePrefix();
}
}
void onData(Block & block)
{
if (written_progress_chars)
@ -1276,6 +1272,14 @@ private:
}
void onLogData(Block & block)
{
initLogsOutputStream();
logs_out_stream->write(block);
logs_out_stream->flush();
}
void onTotals(Block & block)
{
initBlockOutputStream(block);
@ -1436,6 +1440,9 @@ private:
if (block_out_stream)
block_out_stream->writeSuffix();
if (logs_out_stream)
logs_out_stream->writeSuffix();
resetOutput();
if (is_interactive && !written_first_block)
@ -1511,7 +1518,9 @@ public:
ioctl(0, TIOCGWINSZ, &terminal_size);
unsigned line_length = boost::program_options::options_description::m_default_line_length;
namespace po = boost::program_options;
unsigned line_length = po::options_description::m_default_line_length;
unsigned min_description_length = line_length / 2;
if (!stdin_is_not_tty)
{
@ -1519,28 +1528,28 @@ public:
min_description_length = std::min(min_description_length, line_length - 2);
}
#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, boost::program_options::value<std::string> (), DESCRIPTION)
#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, po::value<std::string> (), DESCRIPTION)
/// Main commandline options related to client functionality and all parameters from Settings.
boost::program_options::options_description main_description("Main options", line_length, min_description_length);
po::options_description main_description("Main options", line_length, min_description_length);
main_description.add_options()
("help", "produce help message")
("config-file,c", boost::program_options::value<std::string>(), "config-file path")
("host,h", boost::program_options::value<std::string>()->default_value("localhost"), "server host")
("port", boost::program_options::value<int>()->default_value(9000), "server port")
("config-file,c", po::value<std::string>(), "config-file path")
("host,h", po::value<std::string>()->default_value("localhost"), "server host")
("port", po::value<int>()->default_value(9000), "server port")
("secure,s", "secure")
("user,u", boost::program_options::value<std::string>()->default_value("default"), "user")
("password", boost::program_options::value<std::string>(), "password")
("user,u", po::value<std::string>()->default_value("default"), "user")
("password", po::value<std::string>(), "password")
("ask-password", "ask-password")
("query_id", boost::program_options::value<std::string>(), "query_id")
("query,q", boost::program_options::value<std::string>(), "query")
("database,d", boost::program_options::value<std::string>(), "database")
("pager", boost::program_options::value<std::string>(), "pager")
("query_id", po::value<std::string>(), "query_id")
("query,q", po::value<std::string>(), "query")
("database,d", po::value<std::string>(), "database")
("pager", po::value<std::string>(), "pager")
("multiline,m", "multiline")
("multiquery,n", "multiquery")
("format,f", po::value<std::string>(), "default output format")
("testmode,T", "enable test hints in comments")
("ignore-error", "do not stop processing in multiquery mode")
("format,f", boost::program_options::value<std::string>(), "default output format")
("vertical,E", "vertical output format, same as --format=Vertical or FORMAT Vertical or \\G at end of command")
("time,t", "print query execution time to stderr in non-interactive mode (for benchmarks)")
("stacktrace", "print stack traces of exceptions")
@ -1548,27 +1557,29 @@ public:
("version,V", "print version information and exit")
("version-clean", "print version in machine-readable format and exit")
("echo", "in batch mode, print query before execution")
("max_client_network_bandwidth", boost::program_options::value<int>(), "the maximum speed of data exchange over the network for the client in bytes per second.")
("compression", boost::program_options::value<bool>(), "enable or disable compression")
("max_client_network_bandwidth", po::value<int>(), "the maximum speed of data exchange over the network for the client in bytes per second.")
("compression", po::value<bool>(), "enable or disable compression")
("log-level", po::value<std::string>(), "client log level")
("server_logs_file", po::value<std::string>(), "put server logs into specified file")
APPLY_FOR_SETTINGS(DECLARE_SETTING)
;
#undef DECLARE_SETTING
/// Commandline options related to external tables.
boost::program_options::options_description external_description("External tables options");
po::options_description external_description("External tables options");
external_description.add_options()
("file", boost::program_options::value<std::string>(), "data file or - for stdin")
("name", boost::program_options::value<std::string>()->default_value("_data"), "name of the table")
("format", boost::program_options::value<std::string>()->default_value("TabSeparated"), "data format")
("structure", boost::program_options::value<std::string>(), "structure")
("types", boost::program_options::value<std::string>(), "types")
("file", po::value<std::string>(), "data file or - for stdin")
("name", po::value<std::string>()->default_value("_data"), "name of the table")
("format", po::value<std::string>()->default_value("TabSeparated"), "data format")
("structure", po::value<std::string>(), "structure")
("types", po::value<std::string>(), "types")
;
/// Parse main commandline options.
boost::program_options::parsed_options parsed = boost::program_options::command_line_parser(
po::parsed_options parsed = po::command_line_parser(
common_arguments.size(), common_arguments.data()).options(main_description).run();
boost::program_options::variables_map options;
boost::program_options::store(parsed, options);
po::variables_map options;
po::store(parsed, options);
if (options.count("version") || options.count("V"))
{
@ -1591,14 +1602,17 @@ public:
exit(0);
}
if (options.count("log-level"))
Poco::Logger::root().setLevel(options["log-level"].as<std::string>());
size_t number_of_external_tables_with_stdin_source = 0;
for (size_t i = 0; i < external_tables_arguments.size(); ++i)
{
/// Parse commandline options related to external tables.
boost::program_options::parsed_options parsed = boost::program_options::command_line_parser(
po::parsed_options parsed = po::command_line_parser(
external_tables_arguments[i].size(), external_tables_arguments[i].data()).options(external_description).run();
boost::program_options::variables_map external_options;
boost::program_options::store(parsed, external_options);
po::variables_map external_options;
po::store(parsed, external_options);
try
{
@ -1672,6 +1686,8 @@ public:
max_client_network_bandwidth = options["max_client_network_bandwidth"].as<int>();
if (options.count("compression"))
config().setBool("compression", options["compression"].as<bool>());
if (options.count("server_logs_file"))
server_logs_file = options["server_logs_file"].as<std::string>();
}
};
@ -1691,6 +1707,11 @@ int mainEntryClickHouseClient(int argc, char ** argv)
std::cerr << "Bad arguments: " << e.what() << std::endl;
return 1;
}
catch (...)
{
std::cerr << DB::getCurrentExceptionMessage(true) << std::endl;
return 1;
}
return client.run();
}

118
dbms/programs/client/TestHint.h Normal file
View File

@ -0,0 +1,118 @@
#pragma once
#include <memory>
#include <sstream>
#include <iostream>
#include <Core/Types.h>
#include <Common/Exception.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
/// Checks expected server and client error codes in testmode.
/// To enable it add special comment after the query: "-- { serverError 60 }" or "-- { clientError 20 }".
class TestHint
{
public:
TestHint(bool enabled_, const String & query)
: enabled(enabled_)
{
if (!enabled_)
return;
/// TODO: This is absolutely wrong. Fragment may be contained inside string literal.
size_t pos = query.find("--");
if (pos != String::npos && query.find("--", pos + 2) != String::npos)
return; /// It's not last comment. Hint belongs to commented query. /// TODO Absolutely wrong: there maybe the following comment for the next query.
if (pos != String::npos)
{
/// TODO: This is also wrong. Comment may already have ended by line break.
pos = query.find('{', pos + 2);
if (pos != String::npos)
{
String hint = query.substr(pos + 1);
/// TODO: And this is wrong for the same reason.
pos = hint.find('}');
hint.resize(pos);
parse(hint);
}
}
}
/// @returns true if it's possible to continue without reconnect
bool checkActual(int & actual_server_error, int & actual_client_error,
bool & got_exception, std::unique_ptr<Exception> & last_exception) const
{
if (!enabled)
return true;
if (allErrorsExpected(actual_server_error, actual_client_error))
{
got_exception = false;
last_exception.reset();
actual_server_error = 0;
actual_client_error = 0;
return false;
}
if (lostExpectedError(actual_server_error, actual_client_error))
{
std::cerr << "Success when error expected. It expects server error "
<< server_error << ", client error " << client_error << "." << std::endl;
got_exception = true;
last_exception = std::make_unique<Exception>("Success when error expected", ErrorCodes::LOGICAL_ERROR); /// return error to OS
return false;
}
return true;
}
int serverError() const { return server_error; }
int clientError() const { return client_error; }
private:
bool enabled = false;
int server_error = 0;
int client_error = 0;
void parse(const String & hint)
{
std::stringstream ss;
ss << hint;
while (!ss.eof())
{
String item;
ss >> item;
if (item.empty())
break;
if (item == "serverError")
ss >> server_error;
else if (item == "clientError")
ss >> client_error;
}
}
bool allErrorsExpected(int actual_server_error, int actual_client_error) const
{
return (server_error || client_error) && (server_error == actual_server_error) && (client_error == actual_client_error);
}
bool lostExpectedError(int actual_server_error, int actual_client_error) const
{
return (server_error && !actual_server_error) || (client_error && !actual_client_error);
}
};
}

4
dbms/programs/copier/ClusterCopier.cpp
View File

@ -30,6 +30,7 @@
#include <Common/ClickHouseRevision.h>
#include <Common/formatReadable.h>
#include <Common/DNSResolver.h>
#include <Common/CurrentThread.h>
#include <Common/escapeForFileName.h>
#include <Common/getNumberOfPhysicalCPUCores.h>
#include <Client/Connection.h>
@ -2143,6 +2144,9 @@ void ClusterCopierApp::mainImpl()
context->addDatabase(default_database, std::make_shared<DatabaseMemory>(default_database));
context->setCurrentDatabase(default_database);
/// Initialize query scope just in case.
CurrentThread::QueryScope query_scope(*context);
auto copier = std::make_unique<ClusterCopier>(task_path, host_id, default_database, *context);
copier->setSafeMode(is_safe_mode);
copier->setCopyFaultProbability(copy_fault_probability);

4
dbms/programs/local/LocalServer.cpp
View File

@ -25,6 +25,7 @@
#include <Parsers/IAST.h>
#include <common/ErrorHandlers.h>
#include <Common/StatusFile.h>
#include <Common/ThreadStatus.h>
#include <Functions/registerFunctions.h>
#include <AggregateFunctions/registerAggregateFunctions.h>
#include <TableFunctions/registerTableFunctions.h>
@ -270,6 +271,9 @@ void LocalServer::processQueries()
context->setCurrentQueryId("");
applyCmdSettings(*context);
/// Use the same query_id (and thread group) for all queries
CurrentThread::QueryScope query_scope_holder(*context);
bool echo_query = config().hasOption("echo") || config().hasOption("verbose");
std::exception_ptr exception;

24
dbms/programs/odbc-bridge/CMakeLists.txt
View File

@ -1,12 +1,30 @@
add_library (clickhouse-odbc-bridge-lib
Handlers.cpp
PingHandler.cpp
MainHandler.cpp
ColumnInfoHandler.cpp
HandlerFactory.cpp
ODBCBridge.cpp
)
validateODBCConnectionString.cpp
)
target_link_libraries (clickhouse-odbc-bridge-lib clickhouse_common_io daemon)
target_link_libraries (clickhouse-odbc-bridge-lib clickhouse_common_io daemon dbms)
target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
if (USE_POCO_SQLODBC)
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_SQLODBC_LIBRARY})
target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIRS})
endif ()
if (USE_POCO_DATAODBC)
target_link_libraries (clickhouse-odbc-bridge-lib ${Poco_DataODBC_LIBRARY})
target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIRS})
endif()
if (ENABLE_TESTS)
add_subdirectory (tests)
endif ()
if (CLICKHOUSE_SPLIT_BINARY)
add_executable (clickhouse-odbc-bridge odbc-bridge.cpp)
target_link_libraries (clickhouse-odbc-bridge clickhouse-odbc-bridge-lib)

123
dbms/programs/odbc-bridge/ColumnInfoHandler.cpp Normal file
View File

@ -0,0 +1,123 @@
#include "ColumnInfoHandler.h"
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
#include <Poco/Data/ODBC/ODBCException.h>
#include <Poco/Data/ODBC/SessionImpl.h>
#include <Poco/Data/ODBC/Utility.h>
#include <Poco/Net/HTTPServerRequest.h>
#include <Poco/Net/HTTPServerResponse.h>
#include <DataTypes/DataTypeFactory.h>
#include <IO/WriteBufferFromHTTPServerResponse.h>
#include <IO/WriteHelpers.h>
#include <Common/HTMLForm.h>
#include <common/logger_useful.h>
#include <ext/scope_guard.h>
#include "validateODBCConnectionString.h"
namespace DB
{
namespace
{
DataTypePtr getDataType(SQLSMALLINT type)
{
const auto & factory = DataTypeFactory::instance();
switch (type)
{
case SQL_INTEGER:
return factory.get("Int32");
case SQL_SMALLINT:
return factory.get("Int16");
case SQL_FLOAT:
return factory.get("Float32");
case SQL_REAL:
return factory.get("Float32");
case SQL_DOUBLE:
return factory.get("Float64");
case SQL_DATETIME:
return factory.get("DateTime");
case SQL_TYPE_TIMESTAMP:
return factory.get("DateTime");
case SQL_TYPE_DATE:
return factory.get("Date");
default:
return factory.get("String");
}
}
}
void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response)
{
Poco::Net::HTMLForm params(request, request.stream());
LOG_TRACE(log, "Request URI: " + request.getURI());
auto process_error = [&response, this](const std::string & message) {
response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR);
if (!response.sent())
response.send() << message << std::endl;
LOG_WARNING(log, message);
};
if (!params.has("table"))
{
process_error("No 'table' param in request URL");
return;
}
if (!params.has("connection_string"))
{
process_error("No 'connection_string' in request URL");
return;
}
std::string table_name = params.get("table");
std::string connection_string = params.get("connection_string");
LOG_TRACE(log, "Will fetch info for table '" << table_name << "'");
LOG_TRACE(log, "Got connection str '" << connection_string << "'");
try
{
Poco::Data::ODBC::SessionImpl session(validateODBCConnectionString(connection_string), DBMS_DEFAULT_CONNECT_TIMEOUT_SEC);
SQLHDBC hdbc = session.dbc().handle();
SQLHSTMT hstmt = nullptr;
if (Poco::Data::ODBC::Utility::isError(SQLAllocStmt(hdbc, &hstmt)))
throw Poco::Data::ODBC::ODBCException("Could not allocate connection handle.");
SCOPE_EXIT(SQLFreeStmt(hstmt, SQL_DROP));
/// TODO Why not do SQLColumns instead?
std::string query = "SELECT * FROM " + table_name + " WHERE 1 = 0";
if (Poco::Data::ODBC::Utility::isError(Poco::Data::ODBC::SQLPrepare(hstmt, reinterpret_cast<SQLCHAR *>(&query[0]), query.size())))
throw Poco::Data::ODBC::DescriptorException(session.dbc());
if (Poco::Data::ODBC::Utility::isError(SQLExecute(hstmt)))
throw Poco::Data::ODBC::StatementException(hstmt);
SQLSMALLINT cols = 0;
if (Poco::Data::ODBC::Utility::isError(SQLNumResultCols(hstmt, &cols)))
throw Poco::Data::ODBC::StatementException(hstmt);
/// TODO cols not checked
NamesAndTypesList columns;
for (SQLSMALLINT ncol = 1; ncol <= cols; ++ncol)
{
SQLSMALLINT type = 0;
/// TODO Why 301?
SQLCHAR column_name[301];
/// TODO Result is not checked.
Poco::Data::ODBC::SQLDescribeCol(hstmt, ncol, column_name, sizeof(column_name), NULL, &type, NULL, NULL, NULL);
columns.emplace_back(reinterpret_cast<char *>(column_name), getDataType(type));
}
WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout);
writeStringBinary(columns.toString(), out);
}
catch (...)
{
process_error("Error getting columns from ODBC '" + getCurrentExceptionMessage(false) + "'");
tryLogCurrentException(log);
}
}
}
#endif

30
dbms/programs/odbc-bridge/ColumnInfoHandler.h Normal file
View File

@ -0,0 +1,30 @@
#pragma once
#include <Common/config.h>
#include <Interpreters/Context.h>
#include <Poco/Logger.h>
#include <Poco/Net/HTTPRequestHandler.h>
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
/** The structure of the table is taken from the query "SELECT * FROM table WHERE 1=0".
* TODO: It would be much better to utilize ODBC methods dedicated for columns description.
* If there is no such table, an exception is thrown.
*/
namespace DB
{
class ODBCColumnsInfoHandler : public Poco::Net::HTTPRequestHandler
{
public:
ODBCColumnsInfoHandler(size_t keep_alive_timeout_, std::shared_ptr<Context> context_)
: log(&Poco::Logger::get("ODBCColumnsInfoHandler")), keep_alive_timeout(keep_alive_timeout_), context(context_)
{
}
void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override;
private:
Poco::Logger * log;
size_t keep_alive_timeout;
std::shared_ptr<Context> context;
};
}
#endif

19
dbms/programs/odbc-bridge/HandlerFactory.cpp
View File

@ -1,4 +1,6 @@
#include "HandlerFactory.h"
#include "PingHandler.h"
#include "ColumnInfoHandler.h"
#include <Common/HTMLForm.h>
#include <Poco/Ext/SessionPoolHelpers.h>
@ -9,15 +11,24 @@ namespace DB
{
Poco::Net::HTTPRequestHandler * HandlerFactory::createRequestHandler(const Poco::Net::HTTPServerRequest & request)
{
const auto & uri = request.getURI();
LOG_TRACE(log, "Request URI: " + uri);
Poco::URI uri{request.getURI()};
LOG_TRACE(log, "Request URI: " + uri.toString());
if (uri == "/ping" && request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET)
if (uri.getPath() == "/ping" && request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET)
return new PingHandler(keep_alive_timeout);
if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST)
return new ODBCHandler(pool_map, keep_alive_timeout, context);
{
if (uri.getPath() == "/columns_info")
#if USE_POCO_SQLODBC || USE_POCO_DATAODBC
return new ODBCColumnsInfoHandler(keep_alive_timeout, context);
#else
return nullptr;
#endif
else
return new ODBCHandler(pool_map, keep_alive_timeout, context);
}
return nullptr;
}
}

5
dbms/programs/odbc-bridge/HandlerFactory.h
View File

@ -3,7 +3,8 @@
#include <Poco/Logger.h>
#include <Poco/Net/HTTPRequestHandler.h>
#include <Poco/Net/HTTPRequestHandlerFactory.h>
#include "Handlers.h"
#include "MainHandler.h"
#include "ColumnInfoHandler.h"
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
@ -13,7 +14,7 @@
namespace DB
{
/** Factory for '/ping' and '/' handlers.
/** Factory for '/ping', '/' and '/columns_info' handlers.
* Also stores Session pools for ODBC connections
*/
class HandlerFactory : public Poco::Net::HTTPRequestHandlerFactory

32
dbms/programs/odbc-bridge/Handlers.cpp → dbms/programs/odbc-bridge/MainHandler.cpp
View File

@ -1,30 +1,24 @@
#include "Handlers.h"
#include <Common/HTMLForm.h>
#include "MainHandler.h"
#include "validateODBCConnectionString.h"
#include <memory>
#include <DataStreams/IBlockOutputStream.h>
#include <DataStreams/copyData.h>
#include <DataTypes/DataTypeFactory.h>
#include <Dictionaries/ODBCBlockInputStream.h>
#include <Formats/BinaryRowInputStream.h>
#include <Formats/FormatFactory.h>
#include <IO/ReadBufferFromIStream.h>
#include <IO/WriteBufferFromHTTPServerResponse.h>
#include <IO/WriteHelpers.h>
#include <Interpreters/Context.h>
#include <boost/algorithm/string.hpp>
#include <boost/tokenizer.hpp>
#include <Poco/Ext/SessionPoolHelpers.h>
#include <Poco/Net/HTTPServerRequest.h>
#include <Poco/Net/HTTPServerResponse.h>
#include <Common/HTMLForm.h>
#include <common/logger_useful.h>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_REQUEST_PARAMETER;
}
namespace
{
std::unique_ptr<Block> parseColumns(std::string && column_string)
@ -44,7 +38,7 @@ ODBCHandler::PoolPtr ODBCHandler::getPool(const std::string & connection_str)
if (!pool_map->count(connection_str))
{
pool_map->emplace(connection_str, createAndCheckResizePocoSessionPool([connection_str] {
return std::make_shared<Poco::Data::SessionPool>("ODBC", connection_str);
return std::make_shared<Poco::Data::SessionPool>("ODBC", validateODBCConnectionString(connection_str));
}));
}
return pool_map->at(connection_str);
@ -129,18 +123,4 @@ void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Ne
tryLogCurrentException(log);
}
}
void PingHandler::handleRequest(Poco::Net::HTTPServerRequest & /*request*/, Poco::Net::HTTPServerResponse & response)
{
try
{
setResponseDefaultHeaders(response, keep_alive_timeout);
const char * data = "Ok.\n";
response.sendBuffer(data, strlen(data));
}
catch (...)
{
tryLogCurrentException("PingHandler");
}
}
}

11
dbms/programs/odbc-bridge/Handlers.h → dbms/programs/odbc-bridge/MainHandler.h
View File

@ -46,15 +46,4 @@ private:
PoolPtr getPool(const std::string & connection_str);
};
/** Simple ping handler, answers "Ok." to GET request
*/
class PingHandler : public Poco::Net::HTTPRequestHandler
{
public:
PingHandler(size_t keep_alive_timeout_) : keep_alive_timeout(keep_alive_timeout_) {}
void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override;
private:
size_t keep_alive_timeout;
};
}

22
dbms/programs/odbc-bridge/PingHandler.cpp Normal file
View File

@ -0,0 +1,22 @@
#include "PingHandler.h"
#include <Poco/Net/HTTPServerRequest.h>
#include <Poco/Net/HTTPServerResponse.h>
#include <Common/Exception.h>
#include <IO/HTTPCommon.h>
namespace DB
{
void PingHandler::handleRequest(Poco::Net::HTTPServerRequest & /*request*/, Poco::Net::HTTPServerResponse & response)
{
try
{
setResponseDefaultHeaders(response, keep_alive_timeout);
const char * data = "Ok.\n";
response.sendBuffer(data, strlen(data));
}
catch (...)
{
tryLogCurrentException("PingHandler");
}
}
}

17
dbms/programs/odbc-bridge/PingHandler.h Normal file
View File

@ -0,0 +1,17 @@
#pragma once
#include <Poco/Net/HTTPRequestHandler.h>
namespace DB
{
/** Simple ping handler, answers "Ok." to GET request
*/
class PingHandler : public Poco::Net::HTTPRequestHandler
{
public:
PingHandler(size_t keep_alive_timeout_) : keep_alive_timeout(keep_alive_timeout_) {}
void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override;
private:
size_t keep_alive_timeout;
};
}

2
dbms/programs/odbc-bridge/tests/CMakeLists.txt Normal file
View File

@ -0,0 +1,2 @@
add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp)
target_link_libraries (validate-odbc-connection-string clickhouse-odbc-bridge-lib)

2
dbms/src/Common/tests/validate-odbc-connection-string.cpp → dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.cpp
View File

@ -1,6 +1,6 @@
#include <iostream>
#include <Common/Exception.h>
#include <Common/validateODBCConnectionString.h>
#include "../validateODBCConnectionString.h"
using namespace DB;

0
dbms/src/Common/tests/validate-odbc-connection-string.reference → dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.reference
View File

0
dbms/src/Common/tests/validate-odbc-connection-string.sh → dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.sh
View File

2
dbms/src/Common/validateODBCConnectionString.cpp → dbms/programs/odbc-bridge/validateODBCConnectionString.cpp
View File

@ -5,7 +5,7 @@
#include <common/find_first_symbols.h>
#include <Common/Exception.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/validateODBCConnectionString.h>
#include "validateODBCConnectionString.h"
namespace DB

0
dbms/src/Common/validateODBCConnectionString.h → dbms/programs/odbc-bridge/validateODBCConnectionString.h
View File

74
dbms/programs/server/HTTPHandler.cpp
View File

@ -9,10 +9,11 @@
#include <ext/scope_guard.h>
#include <Common/ExternalTable.h>
#include <Core/ExternalTable.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/escapeForFileName.h>
#include <Common/getFQDNOrHostName.h>
#include <Common/CurrentThread.h>
#include <IO/ReadBufferFromIStream.h>
#include <IO/ZlibInflatingReadBuffer.h>
#include <IO/ReadBufferFromString.h>
@ -208,6 +209,13 @@ void HTTPHandler::processQuery(
Poco::Net::HTTPServerResponse & response,
Output & used_output)
{
Context context = server.context();
context.setGlobalContext(server.context());
/// It will forcibly detach query even if unexpected error ocurred and detachQuery() was not called
/// Normal detaching is happen in BlockIO callbacks
CurrentThread::QueryScope query_scope_holder(context);
LOG_TRACE(log, "Request URI: " << request.getURI());
std::istream & istr = request.stream();
@ -257,14 +265,9 @@ void HTTPHandler::processQuery(
}
std::string query_id = params.get("query_id", "");
const auto & config = server.config();
Context context = server.context();
context.setGlobalContext(server.context());
context.setUser(user, password, request.clientAddress(), quota_key);
context.setCurrentQueryId(query_id);
CurrentThread::attachQueryContext(context);
/// The user could specify session identifier and session timeout.
/// It allows to modify settings, create temporary tables and reuse them in subsequent requests.
@ -273,6 +276,7 @@ void HTTPHandler::processQuery(
String session_id;
std::chrono::steady_clock::duration session_timeout;
bool session_is_set = params.has("session_id");
const auto & config = server.config();
if (session_is_set)
{
@ -421,34 +425,45 @@ void HTTPHandler::processQuery(
std::unique_ptr<ReadBuffer> in;
// Used in case of POST request with form-data, but it not to be expectd to be deleted after that scope
static const NameSet reserved_param_names{"query", "compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace",
"buffer_size", "wait_end_of_query", "session_id", "session_timeout", "session_check"};
Names reserved_param_suffixes;
auto param_could_be_skipped = [&] (const String & name)
{
if (reserved_param_names.count(name))
return true;
for (const String & suffix : reserved_param_suffixes)
{
if (endsWith(name, suffix))
return true;
}
return false;
};
/// Used in case of POST request with form-data, but it isn't expected to be deleted after that scope.
std::string full_query;
/// Support for "external data for query processing".
if (startsWith(request.getContentType().data(), "multipart/form-data"))
{
ExternalTablesHandler handler(context, params);
/// Params are of both form params POST and uri (GET params)
params.load(request, istr, handler);
for (const auto & it : params)
{
if (it.first == "query")
{
full_query += it.second;
}
}
in = std::make_unique<ReadBufferFromString>(full_query);
/// Skip unneeded parameters to avoid confusing them later with context settings or query parameters.
reserved_param_suffixes.emplace_back("_format");
reserved_param_suffixes.emplace_back("_types");
reserved_param_suffixes.emplace_back("_structure");
/// Erase unneeded parameters to avoid confusing them later with context settings or query
/// parameters.
for (const auto & it : handler.names)
{
params.erase(it + "_format");
params.erase(it + "_types");
params.erase(it + "_structure");
}
/// Params are of both form params POST and uri (GET params)
for (const auto & it : params)
if (it.first == "query")
full_query += it.second;
in = std::make_unique<ReadBufferFromString>(full_query);
}
else
in = std::make_unique<ConcatReadBuffer>(*in_param, *in_post_maybe_compressed);
@ -475,11 +490,6 @@ void HTTPHandler::processQuery(
auto readonly_before_query = settings.readonly;
NameSet reserved_param_names{"query", "compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace",
"buffer_size", "wait_end_of_query",
"session_id", "session_timeout", "session_check"
};
for (auto it = params.begin(); it != params.end(); ++it)
{
if (it->first == "database")
@ -490,7 +500,7 @@ void HTTPHandler::processQuery(
{
context.setDefaultFormat(it->second);
}
else if (reserved_param_names.find(it->first) != reserved_param_names.end())
else if (param_could_be_skipped(it->first))
{
}
else

2
dbms/programs/server/MetricsTransmitter.cpp
View File

@ -81,7 +81,7 @@ void MetricsTransmitter::transmit(std::vector<ProfileEvents::Count> & prev_count
{
for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i)
{
const auto counter = ProfileEvents::counters[i].load(std::memory_order_relaxed);
const auto counter = ProfileEvents::global_counters[i].load(std::memory_order_relaxed);
const auto counter_increment = counter - prev_counters[i];
prev_counters[i] = counter;

8
dbms/programs/server/Server.cpp
View File

@ -22,6 +22,7 @@
#include <Common/getFQDNOrHostName.h>
#include <Common/getMultipleKeysFromConfig.h>
#include <Common/getNumberOfPhysicalCPUCores.h>
#include <Common/TaskStatsInfoGetter.h>
#include <IO/HTTPCommon.h>
#include <Interpreters/AsynchronousMetrics.h>
#include <Interpreters/DDLWorker.h>
@ -365,6 +366,13 @@ int Server::main(const std::vector<std::string> & /*args*/)
dns_cache_updater = std::make_unique<DNSCacheUpdater>(*global_context);
}
if (!TaskStatsInfoGetter::checkProcessHasRequiredPermissions())
{
LOG_INFO(log, "It looks like the process has not CAP_NET_ADMIN capability, some performance statistics will be disabled."
" It could happen due to incorrect clickhouse package installation."
" You could resolve the problem manually calling 'sudo setcap cap_net_admin=+ep /usr/bin/clickhouse'");
}
{
Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0);

151
dbms/programs/server/TCPHandler.cpp
View File

@ -1,7 +1,15 @@
#include "TCPHandler.h"
#include <iomanip>
#include <ext/scope_guard.h>
#include <Poco/Net/NetException.h>
#include <daemon/OwnSplitChannel.h>
#include <Common/ClickHouseRevision.h>
#include <Common/CurrentThread.h>
#include <Common/Stopwatch.h>
#include <Common/ClickHouseRevision.h>
#include <Common/Stopwatch.h>
#include <Common/NetException.h>
#include <Common/config_version.h>
#include <IO/Progress.h>
#include <IO/CompressedReadBuffer.h>
#include <IO/CompressedWriteBuffer.h>
@ -15,14 +23,13 @@
#include <Interpreters/executeQuery.h>
#include <Interpreters/Quota.h>
#include <Interpreters/TablesStatus.h>
#include <Interpreters/InternalTextLogsQueue.h>
#include <Storages/StorageMemory.h>
#include <Storages/StorageReplicatedMergeTree.h>
#include <Common/ClickHouseRevision.h>
#include <Common/Stopwatch.h>
#include <Common/ExternalTable.h>
#include <Common/NetException.h>
#include <Common/config_version.h>
#include <ext/scope_guard.h>
#include <Core/ExternalTable.h>
#include "TCPHandler.h"
namespace DB
{
@ -140,13 +147,29 @@ void TCPHandler::runImpl()
if (!receivePacket())
continue;
/// Get blocks of temporary tables
readData(global_settings);
CurrentThread::initializeQuery();
/// Reset the input stream, as we received an empty block while receiving external table data.
/// So, the stream has been marked as cancelled and we can't read from it anymore.
state.block_in.reset();
state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker.
/// Should we send internal logs to client?
if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_LOGS
&& query_context.getSettingsRef().send_logs_level.value != "none")
{
state.logs_queue = std::make_shared<InternalTextLogsQueue>();
state.logs_queue->max_priority = Poco::Logger::parseLevel(query_context.getSettingsRef().send_logs_level.value);
CurrentThread::attachInternalTextLogsQueue(state.logs_queue);
}
query_context.setExternalTablesInitializer([&global_settings, this] (Context & context) {
if (&context != &query_context)
throw Exception("Unexpected context in external tables initializer", ErrorCodes::LOGICAL_ERROR);
/// Get blocks of temporary tables
readData(global_settings);
/// Reset the input stream, as we received an empty block while receiving external table data.
/// So, the stream has been marked as cancelled and we can't read from it anymore.
state.block_in.reset();
state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker.
});
/// Processing Query
state.io = executeQuery(state.query, query_context, false, state.stage);
@ -163,8 +186,9 @@ void TCPHandler::runImpl()
else
processOrdinaryQuery();
sendEndOfStream();
sendLogs();
sendEndOfStream();
state.reset();
}
catch (const Exception & e)
@ -209,7 +233,20 @@ void TCPHandler::runImpl()
try
{
if (exception)
{
try
{
/// Try to send logs to client, but it could be risky too
/// Assume that we can't break output here
sendLogs();
}
catch (...)
{
tryLogCurrentException(log, "Can't send logs to client");
}
sendException(*exception);
}
}
catch (...)
{
@ -220,6 +257,9 @@ void TCPHandler::runImpl()
try
{
/// It will forcibly detach query even if unexpected error ocсurred and detachQuery() was not called
CurrentThread::detachQueryIfNotDetached();
state.reset();
}
catch (...)
@ -252,12 +292,14 @@ void TCPHandler::readData(const Settings & global_settings)
constexpr size_t min_poll_interval = 5000; // 5 ms
size_t poll_interval = std::max(min_poll_interval, std::min(default_poll_interval, current_poll_interval));
while (1)
sendLogs();
while (true)
{
Stopwatch watch(CLOCK_MONOTONIC_COARSE);
/// We are waiting for a packet from the client. Thus, every `POLL_INTERVAL` seconds check whether we need to shut down.
while (1)
while (true)
{
if (static_cast<ReadBufferFromPocoSocket &>(*in).poll(poll_interval))
break;
@ -289,6 +331,8 @@ void TCPHandler::readData(const Settings & global_settings)
/// We accept and process data. And if they are over, then we leave.
if (!receivePacket())
break;
sendLogs();
}
}
@ -346,6 +390,8 @@ void TCPHandler::processOrdinaryQuery()
sendProgress();
}
sendLogs();
if (async_in.poll(query_context.getSettingsRef().interactive_delay / 1000))
{
/// There is the following result block.
@ -368,6 +414,7 @@ void TCPHandler::processOrdinaryQuery()
sendExtremes();
sendProfileInfo();
sendProgress();
sendLogs();
}
sendData(block);
@ -692,11 +739,14 @@ void TCPHandler::initBlockOutput(const Block & block)
{
if (!state.block_out)
{
if (state.compression == Protocol::Compression::Enable)
state.maybe_compressed_out = std::make_shared<CompressedWriteBuffer>(
*out, CompressionSettings(query_context.getSettingsRef()));
else
state.maybe_compressed_out = out;
if (!state.maybe_compressed_out)
{
if (state.compression == Protocol::Compression::Enable)
state.maybe_compressed_out = std::make_shared<CompressedWriteBuffer>(
*out, CompressionSettings(query_context.getSettingsRef()));
else
state.maybe_compressed_out = out;
}
state.block_out = std::make_shared<NativeBlockOutputStream>(
*state.maybe_compressed_out,
@ -705,6 +755,18 @@ void TCPHandler::initBlockOutput(const Block & block)
}
}
void TCPHandler::initLogsBlockOutput(const Block & block)
{
if (!state.logs_block_out)
{
/// Use uncompressed stream since log blocks usually contain only one row
state.logs_block_out = std::make_shared<NativeBlockOutputStream>(
*out,
client_revision,
block.cloneEmpty());
}
}
bool TCPHandler::isQueryCancelled()
{
@ -745,6 +807,7 @@ void TCPHandler::sendData(const Block & block)
initBlockOutput(block);
writeVarUInt(Protocol::Server::Data, *out);
/// Send external table name (empty name is the main table)
writeStringBinary("", *out);
state.block_out->write(block);
@ -753,6 +816,19 @@ void TCPHandler::sendData(const Block & block)
}
void TCPHandler::sendLogData(const Block & block)
{
initLogsBlockOutput(block);
writeVarUInt(Protocol::Server::Log, *out);
/// Send log tag (empty tag is the default tag)
writeStringBinary("", *out);
state.logs_block_out->write(block);
out->next();
}
void TCPHandler::sendException(const Exception & e)
{
writeVarUInt(Protocol::Server::Exception, *out);
@ -784,6 +860,37 @@ void TCPHandler::sendProgress()
}
void TCPHandler::sendLogs()
{
if (!state.logs_queue)
return;
MutableColumns logs_columns;
MutableColumns curr_logs_columns;
size_t rows = 0;
for (; state.logs_queue->tryPop(curr_logs_columns); ++rows)
{
if (rows == 0)
{
logs_columns = std::move(curr_logs_columns);
}
else
{
for (size_t j = 0; j < logs_columns.size(); ++j)
logs_columns[j]->insertRangeFrom(*curr_logs_columns[j], 0, curr_logs_columns[j]->size());
}
}
if (rows > 0)
{
Block block = InternalTextLogsQueue::getSampleBlock();
block.setColumns(std::move(logs_columns));
sendLogData(block);
}
}
void TCPHandler::run()
{
try

12
dbms/programs/server/TCPHandler.h
View File

@ -5,16 +5,18 @@
#include <Common/getFQDNOrHostName.h>
#include <Common/CurrentMetrics.h>
#include <Common/Stopwatch.h>
#include <IO/Progress.h>
#include <Core/Protocol.h>
#include <Core/QueryProcessingStage.h>
#include <DataStreams/BlockIO.h>
#include <IO/Progress.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <DataStreams/BlockIO.h>
#include <Interpreters/InternalTextLogsQueue.h>
#include <Client/TimeoutSetter.h>
#include "IServer.h"
namespace CurrentMetrics
{
extern const Metric TCPConnection;
@ -63,6 +65,9 @@ struct QueryState
/// Timeouts setter for current query
std::unique_ptr<TimeoutSetter> timeout_setter;
/// A queue with internal logs that will be passed to client
InternalTextLogsQueuePtr logs_queue;
BlockOutputStreamPtr logs_block_out;
void reset()
{
@ -140,8 +145,10 @@ private:
void sendHello();
void sendData(const Block & block); /// Write a block to the network.
void sendLogData(const Block & block);
void sendException(const Exception & e);
void sendProgress();
void sendLogs();
void sendEndOfStream();
void sendProfileInfo();
void sendTotals();
@ -150,6 +157,7 @@ private:
/// Creates state.block_in/block_out for blocks read/write, depending on whether compression is enabled.
void initBlockInput();
void initBlockOutput(const Block & block);
void initLogsBlockOutput(const Block & block);
bool isQueryCancelled();

1
dbms/programs/server/config.d/listen.xml Normal file
View File

@ -0,0 +1 @@
<yandex><listen_host>0.0.0.0</listen_host></yandex>

30
dbms/src/AggregateFunctions/AggregateFunctionRetention.cpp Normal file
View File

@ -0,0 +1,30 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/AggregateFunctionRetention.h>
#include <AggregateFunctions/Helpers.h>
#include <AggregateFunctions/FactoryHelpers.h>
namespace DB
{
namespace
{
AggregateFunctionPtr createAggregateFunctionRetention(const std::string & name, const DataTypes & arguments, const Array & params)
{
assertNoParameters(name, params);
if (arguments.size() > AggregateFunctionRetentionData::max_events )
throw Exception("Too many event arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
return std::make_shared<AggregateFunctionRetention>(arguments);
}
}
void registerAggregateFunctionRetention(AggregateFunctionFactory & factory)
{
factory.registerFunction("retention", createAggregateFunctionRetention, AggregateFunctionFactory::CaseInsensitive);
}
}

150
dbms/src/AggregateFunctions/AggregateFunctionRetention.h Normal file
View File

@ -0,0 +1,150 @@
#pragma once
#include <iostream>
#include <sstream>
#include <unordered_set>
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnArray.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeArray.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <Common/ArenaAllocator.h>
#include <Common/typeid_cast.h>
#include <ext/range.h>
#include <bitset>
#include <AggregateFunctions/IAggregateFunction.h>
namespace DB
{
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
}
struct AggregateFunctionRetentionData
{
static constexpr auto max_events = 32;
using Events = std::bitset<max_events>;
Events events;
void add(UInt8 event)
{
events.set(event);
}
void merge(const AggregateFunctionRetentionData & other)
{
events |= other.events;
}
void serialize(WriteBuffer & buf) const
{
UInt32 event_value = events.to_ulong();
writeBinary(event_value, buf);
}
void deserialize(ReadBuffer & buf)
{
UInt32 event_value;
readBinary(event_value, buf);
events = event_value;
}
};
/**
* The max size of events is 32, that's enough for retention analytics
*
* Usage:
* - retention(cond1, cond2, cond3, ....)
* - returns [cond1_flag, cond1_flag && cond2_flag, cond1_flag && cond3_flag, ...]
*/
class AggregateFunctionRetention final
: public IAggregateFunctionDataHelper<AggregateFunctionRetentionData, AggregateFunctionRetention>
{
private:
UInt8 events_size;
public:
String getName() const override
{
return "retention";
}
AggregateFunctionRetention(const DataTypes & arguments)
{
for (const auto i : ext::range(0, arguments.size()))
{
auto cond_arg = arguments[i].get();
if (!typeid_cast<const DataTypeUInt8 *>(cond_arg))
throw Exception{"Illegal type " + cond_arg->getName() + " of argument " + toString(i) + " of aggregate function "
+ getName() + ", must be UInt8",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
}
events_size = arguments.size();
}
DataTypePtr getReturnType() const override
{
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt8>());
}
void add(AggregateDataPtr place, const IColumn ** columns, const size_t row_num, Arena *) const override
{
for (const auto i : ext::range(0, events_size))
{
auto event = static_cast<const ColumnVector<UInt8> *>(columns[i])->getData()[row_num];
if (event)
{
this->data(place).add(i);
break;
}
}
}
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
{
this->data(place).merge(this->data(rhs));
}
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
{
this->data(place).serialize(buf);
}
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
{
this->data(place).deserialize(buf);
}
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override
{
auto & data_to = static_cast<ColumnArray &>(to).getData();
auto & offsets_to = static_cast<ColumnArray &>(to).getOffsets();
const bool first_flag = this->data(place).events.test(0);
data_to.insert(first_flag ? Field(static_cast<UInt64>(1)) : Field(static_cast<UInt64>(0)));
for (const auto i : ext::range(1, events_size))
{
if (first_flag && this->data(place).events.test(i))
data_to.insert(Field(static_cast<UInt64>(1)));
else
data_to.insert(Field(static_cast<UInt64>(0)));
}
offsets_to.push_back(offsets_to.size() == 0 ? events_size : offsets_to.back() + events_size);
}
const char * getHeaderFilePath() const override
{
return __FILE__;
}
};
}

2
dbms/src/AggregateFunctions/registerAggregateFunctions.cpp
View File

@ -34,6 +34,7 @@ void registerAggregateFunctionCombinatorMerge(AggregateFunctionCombinatorFactory
void registerAggregateFunctionCombinatorNull(AggregateFunctionCombinatorFactory &);
void registerAggregateFunctionHistogram(AggregateFunctionFactory & factory);
void registerAggregateFunctionRetention(AggregateFunctionFactory & factory);
void registerAggregateFunctions()
{
@ -59,6 +60,7 @@ void registerAggregateFunctions()
registerAggregateFunctionsBitwise(factory);
registerAggregateFunctionsMaxIntersections(factory);
registerAggregateFunctionHistogram(factory);
registerAggregateFunctionRetention(factory);
}
{

87
dbms/src/Client/Connection.cpp
View File

@ -114,6 +114,7 @@ void Connection::disconnect()
//LOG_TRACE(log_wrapper.get(), "Disconnecting");
in = nullptr;
last_input_packet_type.reset();
out = nullptr; // can write to socket
if (socket)
socket->close();
@ -379,6 +380,7 @@ void Connection::sendQuery(
maybe_compressed_in.reset();
maybe_compressed_out.reset();
block_in.reset();
block_logs_in.reset();
block_out.reset();
/// Send empty block which means end of data.
@ -506,20 +508,50 @@ bool Connection::poll(size_t timeout_microseconds)
}
bool Connection::hasReadBufferPendingData() const
bool Connection::hasReadPendingData() const
{
return static_cast<const ReadBufferFromPocoSocket &>(*in).hasPendingData();
return last_input_packet_type.has_value() || static_cast<const ReadBufferFromPocoSocket &>(*in).hasPendingData();
}
std::optional<UInt64> Connection::checkPacket(size_t timeout_microseconds)
{
if (last_input_packet_type.has_value())
return last_input_packet_type;
if (hasReadPendingData() || poll(timeout_microseconds))
{
// LOG_TRACE(log_wrapper.get(), "Receiving packet type");
UInt64 packet_type;
readVarUInt(packet_type, *in);
last_input_packet_type.emplace(packet_type);
return last_input_packet_type;
}
return {};
}
Connection::Packet Connection::receivePacket()
{
//LOG_TRACE(log_wrapper.get(), "Receiving packet");
try
{
Packet res;
readVarUInt(res.type, *in);
/// Have we already read packet type?
if (last_input_packet_type)
{
res.type = *last_input_packet_type;
last_input_packet_type.reset();
}
else
{
//LOG_TRACE(log_wrapper.get(), "Receiving packet type");
readVarUInt(res.type, *in);
}
//LOG_TRACE(log_wrapper.get(), "Receiving packet " << res.type << " " << Protocol::Server::toString(res.type));
switch (res.type)
{
@ -549,6 +581,10 @@ Connection::Packet Connection::receivePacket()
res.block = receiveData();
return res;
case Protocol::Server::Log:
res.block = receiveLogData();
return res;
case Protocol::Server::EndOfStream:
return res;
@ -576,14 +612,26 @@ Block Connection::receiveData()
//LOG_TRACE(log_wrapper.get(), "Receiving data");
initBlockInput();
return receiveDataImpl(block_in);
}
Block Connection::receiveLogData()
{
initBlockLogsInput();
return receiveDataImpl(block_logs_in);
}
Block Connection::receiveDataImpl(BlockInputStreamPtr & stream)
{
String external_table_name;
readStringBinary(external_table_name, *in);
size_t prev_bytes = in->count();
/// Read one block from network.
Block res = block_in->read();
Block res = stream->read();
if (throttler)
throttler->add(in->count() - prev_bytes);
@ -592,20 +640,39 @@ Block Connection::receiveData()
}
void Connection::initInputBuffers()
{
}
void Connection::initBlockInput()
{
if (!block_in)
{
if (compression == Protocol::Compression::Enable)
maybe_compressed_in = std::make_shared<CompressedReadBuffer>(*in);
else
maybe_compressed_in = in;
if (!maybe_compressed_in)
{
if (compression == Protocol::Compression::Enable)
maybe_compressed_in = std::make_shared<CompressedReadBuffer>(*in);
else
maybe_compressed_in = in;
}
block_in = std::make_shared<NativeBlockInputStream>(*maybe_compressed_in, server_revision);
}
}
void Connection::initBlockLogsInput()
{
if (!block_logs_in)
{
/// Have to return superset of SystemLogsQueue::getSampleBlock() columns
block_logs_in = std::make_shared<NativeBlockInputStream>(*in, server_revision);
}
}
void Connection::setDescription()
{
auto resolved_address = getResolvedAddress();

13
dbms/src/Client/Connection.h
View File

@ -23,6 +23,7 @@
#include <Interpreters/TablesStatus.h>
#include <atomic>
#include <optional>
namespace DB
@ -138,7 +139,10 @@ public:
bool poll(size_t timeout_microseconds = 0);
/// Check, if has data in read buffer.
bool hasReadBufferPendingData() const;
bool hasReadPendingData() const;
/// Checks if there is input data in connection and reads packet ID.
std::optional<UInt64> checkPacket(size_t timeout_microseconds = 0);
/// Receive packet from server.
Packet receivePacket();
@ -195,6 +199,7 @@ private:
std::unique_ptr<Poco::Net::StreamSocket> socket;
std::shared_ptr<ReadBuffer> in;
std::shared_ptr<WriteBuffer> out;
std::optional<UInt64> last_input_packet_type;
String query_id;
Protocol::Compression compression; /// Enable data compression for communication.
@ -214,6 +219,7 @@ private:
/// From where to read query execution result.
std::shared_ptr<ReadBuffer> maybe_compressed_in;
BlockInputStreamPtr block_in;
BlockInputStreamPtr block_logs_in;
/// Where to write data for INSERT.
std::shared_ptr<WriteBuffer> maybe_compressed_out;
@ -249,11 +255,16 @@ private:
bool ping();
Block receiveData();
Block receiveLogData();
Block receiveDataImpl(BlockInputStreamPtr & stream);
std::unique_ptr<Exception> receiveException();
Progress receiveProgress();
BlockStreamProfileInfo receiveProfileInfo();
void initInputBuffers();
void initBlockInput();
void initBlockLogsInput();
void throwUnexpectedPacket(UInt64 packet_type, const char * expected) const;
};

3
dbms/src/Client/MultiplexedConnections.cpp
View File

@ -247,6 +247,7 @@ Connection::Packet MultiplexedConnections::receivePacketUnlocked()
case Protocol::Server::ProfileInfo:
case Protocol::Server::Totals:
case Protocol::Server::Extremes:
case Protocol::Server::Log:
break;
case Protocol::Server::EndOfStream:
@ -276,7 +277,7 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead
for (const ReplicaState & state : replica_states)
{
Connection * connection = state.connection;
if ((connection != nullptr) && connection->hasReadBufferPendingData())
if ((connection != nullptr) && connection->hasReadPendingData())
read_list.push_back(*connection->socket);
}

2
dbms/src/Columns/ColumnVector.cpp
View File

@ -118,7 +118,7 @@ MutableColumnPtr ColumnVector<T>::cloneResized(size_t size) const
memcpy(&new_col.data[0], &data[0], count * sizeof(data[0]));
if (size > count)
memset(&new_col.data[count], static_cast<int>(value_type()), (size - count) * sizeof(value_type));
memset(static_cast<void *>(&new_col.data[count]), static_cast<int>(value_type()), (size - count) * sizeof(value_type));
}
return std::move(res);

8
dbms/src/Common/ActionBlocker.h
View File

@ -1,14 +1,15 @@
#pragma once
#include <atomic>
#include <memory>
#include <Common/ActionLock.h>
namespace DB
{
/// An atomic variable that is used to block and interrupt certain actions
/// If it is not zero then actions related with it should be considered as interrupted
/// An atomic variable that is used to block and interrupt certain actions.
/// If it is not zero then actions related with it should be considered as interrupted.
/// Uses shared_ptr and the lock uses weak_ptr to be able to "hold" a lock when an object with blocker has already died.
class ActionBlocker
{
public:
@ -33,4 +34,5 @@ private:
CounterPtr counter;
};
}

2
dbms/src/Common/ActionLock.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <memory>
#include <atomic>
#include <Core/Types.h>
namespace DB
{

23
dbms/src/Common/BackgroundSchedulePool.cpp
View File

@ -4,8 +4,11 @@
#include <Common/Exception.h>
#include <Common/setThreadName.h>
#include <Common/Stopwatch.h>
#include <Common/CurrentThread.h>
#include <common/logger_useful.h>
#include <chrono>
#include <ext/scope_guard.h>
namespace CurrentMetrics
{
@ -140,6 +143,12 @@ BackgroundSchedulePool::BackgroundSchedulePool(size_t size)
{
LOG_INFO(&Logger::get("BackgroundSchedulePool"), "Create BackgroundSchedulePool with " << size << " threads");
/// Put all threads of both thread pools to one thread group
/// The master thread exits immediately
CurrentThread::initializeQuery();
thread_group = CurrentThread::getGroup();
CurrentThread::detachQuery();
threads.resize(size);
for (auto & thread : threads)
thread = std::thread([this] { threadFunction(); });
@ -212,9 +221,11 @@ void BackgroundSchedulePool::threadFunction()
{
setThreadName("BackgrSchedPool");
MemoryTracker memory_tracker;
memory_tracker.setMetric(CurrentMetrics::MemoryTrackingInBackgroundSchedulePool);
current_memory_tracker = &memory_tracker;
/// Put all threads to one thread pool
CurrentThread::attachTo(thread_group);
SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); });
CurrentThread::getMemoryTracker().setMetric(CurrentMetrics::MemoryTrackingInBackgroundSchedulePool);
while (!shutdown)
{
@ -224,8 +235,6 @@ void BackgroundSchedulePool::threadFunction()
task_notification.execute();
}
}
current_memory_tracker = nullptr;
}
@ -233,6 +242,10 @@ void BackgroundSchedulePool::delayExecutionThreadFunction()
{
setThreadName("BckSchPoolDelay");
/// Put all threads to one thread pool
CurrentThread::attachTo(thread_group);
SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); });
while (!shutdown)
{
TaskInfoPtr task;

4
dbms/src/Common/BackgroundSchedulePool.h
View File

@ -12,6 +12,7 @@
#include <functional>
#include <boost/noncopyable.hpp>
#include <Common/ZooKeeper/Types.h>
#include <Common/CurrentThread.h>
namespace DB
{
@ -138,6 +139,9 @@ private:
std::thread delayed_thread;
/// Tasks ordered by scheduled time.
DelayedTasks delayed_tasks;
/// Thread group used for profiling purposes
ThreadGroupStatusPtr thread_group;
};
using BackgroundSchedulePoolPtr = std::shared_ptr<BackgroundSchedulePool>;

80
dbms/src/Common/CurrentThread.cpp Normal file
View File

@ -0,0 +1,80 @@
#include "CurrentThread.h"
#include <common/logger_useful.h>
#include <Common/ThreadStatus.h>
#include <Interpreters/ProcessList.h>
#include <Interpreters/Context.h>
#include <Poco/Ext/ThreadNumber.h>
#include <Poco/Logger.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
void CurrentThread::updatePerformanceCounters()
{
get()->updatePerformanceCounters();
}
ThreadStatusPtr CurrentThread::get()
{
#ifndef NDEBUG
if (!current_thread || current_thread.use_count() <= 0)
throw Exception("Thread #" + std::to_string(Poco::ThreadNumber::get()) + " status was not initialized", ErrorCodes::LOGICAL_ERROR);
if (Poco::ThreadNumber::get() != current_thread->thread_number)
throw Exception("Current thread has different thread number", ErrorCodes::LOGICAL_ERROR);
#endif
return current_thread;
}
ProfileEvents::Counters & CurrentThread::getProfileEvents()
{
return current_thread->performance_counters;
}
MemoryTracker & CurrentThread::getMemoryTracker()
{
return current_thread->memory_tracker;
}
void CurrentThread::updateProgressIn(const Progress & value)
{
current_thread->progress_in.incrementPiecewiseAtomically(value);
}
void CurrentThread::updateProgressOut(const Progress & value)
{
current_thread->progress_out.incrementPiecewiseAtomically(value);
}
void CurrentThread::attachInternalTextLogsQueue(const std::shared_ptr<InternalTextLogsQueue> & logs_queue)
{
get()->attachInternalTextLogsQueue(logs_queue);
}
std::shared_ptr<InternalTextLogsQueue> CurrentThread::getInternalTextLogsQueue()
{
/// NOTE: this method could be called at early server startup stage
/// NOTE: this method could be called in ThreadStatus destructor, therefore we make use_count() check just in case
if (!current_thread || current_thread.use_count() <= 0)
return nullptr;
if (current_thread->getCurrentState() == ThreadStatus::ThreadState::Died)
return nullptr;
return current_thread->getInternalTextLogsQueue();
}
ThreadGroupStatusPtr CurrentThread::getGroup()
{
return get()->getThreadGroup();
}
}

83
dbms/src/Common/CurrentThread.h Normal file
View File

@ -0,0 +1,83 @@
#pragma once
#include <memory>
#include <string>
namespace ProfileEvents
{
class Counters;
}
class MemoryTracker;
namespace DB
{
class Context;
class QueryStatus;
class ThreadStatus;
struct Progress;
using ThreadStatusPtr = std::shared_ptr<ThreadStatus>;
class InternalTextLogsQueue;
class ThreadGroupStatus;
using ThreadGroupStatusPtr = std::shared_ptr<ThreadGroupStatus>;
class CurrentThread
{
public:
/// Handler to current thread
static ThreadStatusPtr get();
/// Group to which belongs current thread
static ThreadGroupStatusPtr getGroup();
/// A logs queue used by TCPHandler to pass logs to a client
static void attachInternalTextLogsQueue(const std::shared_ptr<InternalTextLogsQueue> & logs_queue);
static std::shared_ptr<InternalTextLogsQueue> getInternalTextLogsQueue();
/// Makes system calls to update ProfileEvents that contain info from rusage and taskstats
static void updatePerformanceCounters();
static ProfileEvents::Counters & getProfileEvents();
static MemoryTracker & getMemoryTracker();
/// Update read and write rows (bytes) statistics (used in system.query_thread_log)
static void updateProgressIn(const Progress & value);
static void updateProgressOut(const Progress & value);
/// Query management:
/// Call from master thread as soon as possible (e.g. when thread accepted connection)
static void initializeQuery();
/// Sets query_context for current thread group
static void attachQueryContext(Context & query_context);
/// You must call one of these methods when create a query child thread:
/// Add current thread to a group associated with the thread group
static void attachTo(const ThreadGroupStatusPtr & thread_group);
/// Is useful for a ThreadPool tasks
static void attachToIfDetached(const ThreadGroupStatusPtr & thread_group);
/// Update ProfileEvents and dumps info to system.query_thread_log
static void finalizePerformanceCounters();
/// Returns a non-empty string if the thread is attached to a query
static std::string getCurrentQueryID();
/// Non-master threads call this method in destructor automatically
static void detachQuery();
static void detachQueryIfNotDetached();
/// Initializes query with current thread as master thread in constructor, and detaches it in desstructor
struct QueryScope
{
explicit QueryScope(Context & query_context);
~QueryScope();
};
};
}

21
dbms/src/Common/DNSResolver.cpp
View File

@ -7,6 +7,7 @@
#include <Poco/NumberParser.h>
#include <arpa/inet.h>
#include <atomic>
#include <optional>
namespace DB
@ -79,6 +80,10 @@ struct DNSResolver::Impl
{
SimpleCache<decltype(resolveIPAddressImpl), &resolveIPAddressImpl> cache_host;
/// Cached server host name
std::mutex mutex;
std::optional<String> host_name;
/// If disabled, will not make cache lookups, will resolve addresses manually on each call
std::atomic<bool> disable_cache{false};
};
@ -108,6 +113,9 @@ Poco::Net::SocketAddress DNSResolver::resolveAddress(const std::string & host, U
void DNSResolver::dropCache()
{
impl->cache_host.drop();
std::unique_lock lock(impl->mutex);
impl->host_name.reset();
}
void DNSResolver::setDisableCacheFlag(bool is_disabled)
@ -115,6 +123,19 @@ void DNSResolver::setDisableCacheFlag(bool is_disabled)
impl->disable_cache = is_disabled;
}
String DNSResolver::getHostName()
{
if (impl->disable_cache)
return Poco::Net::DNS::hostName();
std::unique_lock lock(impl->mutex);
if (!impl->host_name.has_value())
impl->host_name.emplace(Poco::Net::DNS::hostName());
return *impl->host_name;
}
DNSResolver::~DNSResolver() = default;

3
dbms/src/Common/DNSResolver.h
View File

@ -25,6 +25,9 @@ public:
Poco::Net::SocketAddress resolveAddress(const std::string & host, UInt16 port);
/// Get this server host name
String getHostName();
/// Disables caching
void setDisableCacheFlag(bool is_disabled = true);

2
dbms/src/Common/ErrorCodes.cpp
View File

@ -385,6 +385,8 @@ namespace ErrorCodes
extern const int BAD_REQUEST_PARAMETER = 408;
extern const int EXTERNAL_EXECUTABLE_NOT_FOUND = 409;
extern const int EXTERNAL_SERVER_IS_NOT_RESPONDING = 410;
extern const int PTHREAD_ERROR = 411;
extern const int NETLINK_ERROR = 412;
extern const int KEEPER_EXCEPTION = 999;
extern const int POCO_EXCEPTION = 1000;

225
dbms/src/Common/ExternalTable.h
View File

@ -1,225 +0,0 @@
#pragma once
#include <boost/program_options.hpp>
#include <boost/algorithm/string.hpp>
#include <DataStreams/AsynchronousBlockInputStream.h>
#include <DataTypes/DataTypeFactory.h>
#include <Interpreters/Context.h>
#include <IO/copyData.h>
#include <IO/ReadBufferFromIStream.h>
#include <IO/ReadBufferFromFile.h>
#include <Storages/StorageMemory.h>
#include <Client/Connection.h>
#include <Poco/Net/HTMLForm.h>
#include <Poco/Net/PartHandler.h>
#include <Poco/Net/MessageHeader.h>
#include <Common/HTMLForm.h>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
/// The base class containing the basic information about external table and
/// basic functions for extracting this information from text fields.
class BaseExternalTable
{
public:
std::string file; /// File with data or '-' if stdin
std::string name; /// The name of the table
std::string format; /// Name of the data storage format
/// Description of the table structure: (column name, data type name)
std::vector<std::pair<std::string, std::string>> structure;
std::unique_ptr<ReadBuffer> read_buffer;
Block sample_block;
virtual ~BaseExternalTable() {}
/// Initialize read_buffer, depending on the data source. By default, does nothing.
virtual void initReadBuffer() {}
/// Get the table data - a pair (a thread with the contents of the table, the name of the table)
ExternalTableData getData(const Context & context)
{
initReadBuffer();
initSampleBlock();
ExternalTableData res = std::make_pair(std::make_shared<AsynchronousBlockInputStream>(context.getInputFormat(
format, *read_buffer, sample_block, DEFAULT_BLOCK_SIZE)), name);
return res;
}
protected:
/// Clear all accumulated information
void clean()
{
name = "";
file = "";
format = "";
structure.clear();
sample_block = Block();
read_buffer.reset();
}
/// Function for debugging information output
void write()
{
std::cerr << "file " << file << std::endl;
std::cerr << "name " << name << std::endl;
std::cerr << "format " << format << std::endl;
std::cerr << "structure: \n";
for (size_t i = 0; i < structure.size(); ++i)
std::cerr << "\t" << structure[i].first << " " << structure[i].second << std::endl;
}
static std::vector<std::string> split(const std::string & s, const std::string & d)
{
std::vector<std::string> res;
boost::split(res, s, boost::algorithm::is_any_of(d), boost::algorithm::token_compress_on);
return res;
}
/// Construct the `structure` vector from the text field `structure`
virtual void parseStructureFromStructureField(const std::string & argument)
{
std::vector<std::string> vals = split(argument, " ,");
if (vals.size() & 1)
throw Exception("Odd number of attributes in section structure", ErrorCodes::BAD_ARGUMENTS);
for (size_t i = 0; i < vals.size(); i += 2)
structure.emplace_back(vals[i], vals[i + 1]);
}
/// Construct the `structure` vector from the text field `types`
virtual void parseStructureFromTypesField(const std::string & argument)
{
std::vector<std::string> vals = split(argument, " ,");
for (size_t i = 0; i < vals.size(); ++i)
structure.emplace_back("_" + toString(i + 1), vals[i]);
}
private:
/// Initialize sample_block according to the structure of the table stored in the `structure`
void initSampleBlock()
{
const DataTypeFactory & data_type_factory = DataTypeFactory::instance();
for (size_t i = 0; i < structure.size(); ++i)
{
ColumnWithTypeAndName column;
column.name = structure[i].first;
column.type = data_type_factory.get(structure[i].second);
column.column = column.type->createColumn();
sample_block.insert(std::move(column));
}
}
};
/// Parsing of external table used in the tcp client.
class ExternalTable : public BaseExternalTable
{
public:
void initReadBuffer() override
{
if (file == "-")
read_buffer = std::make_unique<ReadBufferFromFileDescriptor>(STDIN_FILENO);
else
read_buffer = std::make_unique<ReadBufferFromFile>(file);
}
/// Extract parameters from variables_map, which is built on the client command line
ExternalTable(const boost::program_options::variables_map & external_options)
{
if (external_options.count("file"))
file = external_options["file"].as<std::string>();
else
throw Exception("--file field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
if (external_options.count("name"))
name = external_options["name"].as<std::string>();
else
throw Exception("--name field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
if (external_options.count("format"))
format = external_options["format"].as<std::string>();
else
throw Exception("--format field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
if (external_options.count("structure"))
parseStructureFromStructureField(external_options["structure"].as<std::string>());
else if (external_options.count("types"))
parseStructureFromTypesField(external_options["types"].as<std::string>());
else
throw Exception("Neither --structure nor --types have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
}
};
/// Parsing of external table used when sending tables via http
/// The `handlePart` function will be called for each table passed,
/// so it's also necessary to call `clean` at the end of the `handlePart`.
class ExternalTablesHandler : public Poco::Net::PartHandler, BaseExternalTable
{
public:
std::vector<std::string> names;
ExternalTablesHandler(Context & context_, Poco::Net::NameValueCollection params_) : context(context_), params(params_) { }
void handlePart(const Poco::Net::MessageHeader & header, std::istream & stream)
{
/// The buffer is initialized here, not in the virtual function initReadBuffer
read_buffer = std::make_unique<ReadBufferFromIStream>(stream);
/// Retrieve a collection of parameters from MessageHeader
Poco::Net::NameValueCollection content;
std::string label;
Poco::Net::MessageHeader::splitParameters(header.get("Content-Disposition"), label, content);
/// Get parameters
name = content.get("name", "_data");
format = params.get(name + "_format", "TabSeparated");
if (params.has(name + "_structure"))
parseStructureFromStructureField(params.get(name + "_structure"));
else if (params.has(name + "_types"))
parseStructureFromTypesField(params.get(name + "_types"));
else
throw Exception("Neither structure nor types have not been provided for external table " + name + ". Use fields " + name + "_structure or " + name + "_types to do so.", ErrorCodes::BAD_ARGUMENTS);
ExternalTableData data = getData(context);
/// Create table
NamesAndTypesList columns = sample_block.getNamesAndTypesList();
StoragePtr storage = StorageMemory::create(data.second, ColumnsDescription{columns});
storage->startup();
context.addExternalTable(data.second, storage);
BlockOutputStreamPtr output = storage->write(ASTPtr(), context.getSettingsRef());
/// Write data
data.first->readPrefix();
output->writePrefix();
while(Block block = data.first->read())
output->write(block);
data.first->readSuffix();
output->writeSuffix();
names.push_back(name);
/// We are ready to receive the next file, for this we clear all the information received
clean();
}
private:
Context & context;
Poco::Net::NameValueCollection params;
};
}

80
dbms/src/Common/MemoryTracker.cpp
View File

@ -1,11 +1,10 @@
#include "MemoryTracker.h"
#include <common/likely.h>
#include <common/logger_useful.h>
#include <Common/Exception.h>
#include <Common/formatReadable.h>
#include <Common/CurrentThread.h>
#include <IO/WriteHelpers.h>
#include <iomanip>
#include <Common/MemoryTracker.h>
namespace DB
@ -19,7 +18,7 @@ namespace DB
MemoryTracker::~MemoryTracker()
{
if (peak)
if (static_cast<int>(level) < static_cast<int>(VariableContext::Process) && peak)
{
try
{
@ -56,13 +55,16 @@ void MemoryTracker::logPeakMemoryUsage() const
void MemoryTracker::alloc(Int64 size)
{
if (blocker.isCancelled())
return;
/** Using memory_order_relaxed means that if allocations are done simultaneously,
* we allow exception about memory limit exceeded to be thrown only on next allocation.
* So, we allow over-allocations.
*/
Int64 will_be = size + amount.fetch_add(size, std::memory_order_relaxed);
if (!next.load(std::memory_order_relaxed))
if (!parent.load(std::memory_order_relaxed))
CurrentMetrics::add(metric, size);
Int64 current_limit = limit.load(std::memory_order_relaxed);
@ -102,45 +104,62 @@ void MemoryTracker::alloc(Int64 size)
if (will_be > peak.load(std::memory_order_relaxed)) /// Races doesn't matter. Could rewrite with CAS, but not worth.
peak.store(will_be, std::memory_order_relaxed);
if (auto loaded_next = next.load(std::memory_order_relaxed))
if (auto loaded_next = parent.load(std::memory_order_relaxed))
loaded_next->alloc(size);
}
void MemoryTracker::free(Int64 size)
{
Int64 new_amount = amount.fetch_sub(size, std::memory_order_relaxed) - size;
if (blocker.isCancelled())
return;
/** Sometimes, query could free some data, that was allocated outside of query context.
* Example: cache eviction.
* To avoid negative memory usage, we "saturate" amount.
* Memory usage will be calculated with some error.
* NOTE The code is not atomic. Not worth to fix.
*/
if (new_amount < 0)
if (level == VariableContext::Thread)
{
amount.fetch_sub(new_amount);
size += new_amount;
/// Could become negative if memory allocated in this thread is freed in another one
amount.fetch_sub(size, std::memory_order_relaxed);
}
else
{
Int64 new_amount = amount.fetch_sub(size, std::memory_order_relaxed) - size;
/** Sometimes, query could free some data, that was allocated outside of query context.
* Example: cache eviction.
* To avoid negative memory usage, we "saturate" amount.
* Memory usage will be calculated with some error.
* NOTE: The code is not atomic. Not worth to fix.
*/
if (unlikely(new_amount < 0))
{
amount.fetch_sub(new_amount);
size += new_amount;
}
}
if (auto loaded_next = next.load(std::memory_order_relaxed))
if (auto loaded_next = parent.load(std::memory_order_relaxed))
loaded_next->free(size);
else
CurrentMetrics::sub(metric, size);
}
void MemoryTracker::reset()
void MemoryTracker::resetCounters()
{
if (!next.load(std::memory_order_relaxed))
CurrentMetrics::sub(metric, amount.load(std::memory_order_relaxed));
amount.store(0, std::memory_order_relaxed);
peak.store(0, std::memory_order_relaxed);
limit.store(0, std::memory_order_relaxed);
}
void MemoryTracker::reset()
{
if (!parent.load(std::memory_order_relaxed))
CurrentMetrics::sub(metric, amount.load(std::memory_order_relaxed));
resetCounters();
}
void MemoryTracker::setOrRaiseLimit(Int64 value)
{
/// This is just atomic set to maximum.
@ -149,29 +168,26 @@ void MemoryTracker::setOrRaiseLimit(Int64 value)
;
}
#if __APPLE__ && __clang__
__thread MemoryTracker * current_memory_tracker = nullptr;
#else
thread_local MemoryTracker * current_memory_tracker = nullptr;
#endif
namespace CurrentMemoryTracker
{
void alloc(Int64 size)
{
if (current_memory_tracker)
current_memory_tracker->alloc(size);
DB::CurrentThread::getMemoryTracker().alloc(size);
}
void realloc(Int64 old_size, Int64 new_size)
{
if (current_memory_tracker)
current_memory_tracker->alloc(new_size - old_size);
DB::CurrentThread::getMemoryTracker().alloc(new_size - old_size);
}
void free(Int64 size)
{
if (current_memory_tracker)
current_memory_tracker->free(size);
DB::CurrentThread::getMemoryTracker().free(size);
}
}
DB::SimpleActionLock getCurrentMemoryTrackerActionLock()
{
return DB::CurrentThread::getMemoryTracker().blocker.cancel();
}

60
dbms/src/Common/MemoryTracker.h
View File

@ -3,6 +3,8 @@
#include <atomic>
#include <common/Types.h>
#include <Common/CurrentMetrics.h>
#include <Common/SimpleActionBlocker.h>
#include <Common/VariableContext.h>
namespace CurrentMetrics
@ -26,7 +28,7 @@ class MemoryTracker
/// Singly-linked list. All information will be passed to subsequent memory trackers also (it allows to implement trackers hierarchy).
/// In terms of tree nodes it is the list of parents. Lifetime of these trackers should "include" lifetime of current tracker.
std::atomic<MemoryTracker *> next {};
std::atomic<MemoryTracker *> parent {};
/// You could specify custom metric to track memory usage.
CurrentMetrics::Metric metric = CurrentMetrics::MemoryTracking;
@ -35,11 +37,14 @@ class MemoryTracker
const char * description = nullptr;
public:
MemoryTracker() {}
MemoryTracker(Int64 limit_) : limit(limit_) {}
MemoryTracker(VariableContext level = VariableContext::Thread) : level(level) {}
MemoryTracker(Int64 limit_, VariableContext level = VariableContext::Thread) : limit(limit_), level(level) {}
MemoryTracker(MemoryTracker * parent_, VariableContext level = VariableContext::Thread) : parent(parent_), level(level) {}
~MemoryTracker();
VariableContext level;
/** Call the following functions before calling of corresponding operations with memory allocators.
*/
void alloc(Int64 size);
@ -79,9 +84,15 @@ public:
}
/// next should be changed only once: from nullptr to some value.
void setNext(MemoryTracker * elem)
/// NOTE: It is not true in MergeListElement
void setParent(MemoryTracker * elem)
{
next.store(elem, std::memory_order_relaxed);
parent.store(elem, std::memory_order_relaxed);
}
MemoryTracker * getParent()
{
return parent.load(std::memory_order_relaxed);
}
/// The memory consumption could be shown in realtime via CurrentMetrics counter
@ -95,26 +106,21 @@ public:
description = description_;
}
/// Reset the accumulated data.
/// Reset the accumulated data
void resetCounters();
/// Reset the accumulated data and the parent.
void reset();
/// Prints info about peak memory consumption into log.
void logPeakMemoryUsage() const;
/// To be able to temporarily stop memory tracker
DB::SimpleActionBlocker blocker;
};
/** The MemoryTracker object is quite difficult to pass to all places where significant amounts of memory are allocated.
* Therefore, a thread-local pointer to used MemoryTracker is set, or nullptr if MemoryTracker does not need to be used.
* This pointer is set when memory consumption is monitored in current thread.
* So, you just need to pass it to all the threads that handle one request.
*/
#if defined(__APPLE__) && defined(__clang__)
extern __thread MemoryTracker * current_memory_tracker;
#else
extern thread_local MemoryTracker * current_memory_tracker;
#endif
/// Convenience methods, that use current_memory_tracker if it is available.
/// Convenience methods, that use current thread's memory_tracker if it is available.
namespace CurrentMemoryTracker
{
void alloc(Int64 size);
@ -123,20 +129,4 @@ namespace CurrentMemoryTracker
}
#include <boost/noncopyable.hpp>
struct TemporarilyDisableMemoryTracker : private boost::noncopyable
{
MemoryTracker * memory_tracker;
TemporarilyDisableMemoryTracker()
{
memory_tracker = current_memory_tracker;
current_memory_tracker = nullptr;
}
~TemporarilyDisableMemoryTracker()
{
current_memory_tracker = memory_tracker;
}
};
DB::SimpleActionLock getCurrentMemoryTrackerActionLock();

2
dbms/src/Common/NaNUtils.h
View File

@ -50,7 +50,7 @@ std::enable_if_t<std::is_class_v<T>, T> NaNOrZero()
#if 1 /// __int128
template <typename T>
std::enable_if_t<std::is_same_v<T, __int128>, __int128> NaNOrZero()
std::enable_if_t<std::is_same_v<T, __int128> && !std::numeric_limits<T>::is_integer, __int128> NaNOrZero()
{
return __int128(0);
}

45
dbms/src/Common/ODBCBridgeHelper.cpp
View File

@ -1,25 +1,26 @@
#include <Common/ODBCBridgeHelper.h>
#include <sstream>
#include <Common/validateODBCConnectionString.h>
#include <IO/ReadHelpers.h>
#include <IO/ReadWriteBufferFromHTTP.h>
#include <Poco/Net/HTTPRequest.h>
#include <Poco/Path.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <Common/ShellCommand.h>
#include <common/logger_useful.h>
#include <ext/range.h>
namespace DB
{
namespace ErrorCodes
{
extern const int EXTERNAL_SERVER_IS_NOT_RESPONDING;
}
ODBCBridgeHelper::ODBCBridgeHelper(const Context & context_global_, const std::string & connection_string_)
: context_global(context_global_), connection_string(validateODBCConnectionString(connection_string_))
ODBCBridgeHelper::ODBCBridgeHelper(
const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_)
: config(config_), http_timeout(http_timeout_), connection_string(connection_string_)
{
const auto & config = context_global.getConfigRef();
size_t bridge_port = config.getUInt("odbc_bridge.port", DEFAULT_PORT);
std::string bridge_host = config.getString("odbc_bridge.host", DEFAULT_HOST);
@ -30,8 +31,6 @@ ODBCBridgeHelper::ODBCBridgeHelper(const Context & context_global_, const std::s
}
void ODBCBridgeHelper::startODBCBridge() const
{
const auto & config = context_global.getConfigRef();
const auto & settings = context_global.getSettingsRef();
Poco::Path path{config.getString("application.dir", "")};
path.setFileName("clickhouse-odbc-bridge");
@ -42,7 +41,7 @@ void ODBCBridgeHelper::startODBCBridge() const
command << path.toString() << ' ';
command << "--http-port " << config.getUInt("odbc_bridge.port", DEFAULT_PORT) << ' ';
command << "--listen-host " << config.getString("odbc_bridge.listen_host", DEFAULT_HOST) << ' ';
command << "--http-timeout " << settings.http_receive_timeout.value.totalSeconds() << ' ';
command << "--http-timeout " << http_timeout.totalMicroseconds() << ' ';
if (config.has("logger.odbc_bridge_log"))
command << "--log-path " << config.getString("logger.odbc_bridge_log") << ' ';
if (config.has("logger.odbc_bridge_errlog"))
@ -58,12 +57,12 @@ void ODBCBridgeHelper::startODBCBridge() const
cmd->wait();
}
std::vector<std::pair<std::string, std::string>> ODBCBridgeHelper::getURLParams(const NamesAndTypesList & cols, size_t max_block_size) const
std::vector<std::pair<std::string, std::string>> ODBCBridgeHelper::getURLParams(const std::string & cols, size_t max_block_size) const
{
std::vector<std::pair<std::string, std::string>> result;
result.emplace_back("connection_string", connection_string); /// already validated
result.emplace_back("columns", cols.toString());
result.emplace_back("columns", cols);
result.emplace_back("max_block_size", std::to_string(max_block_size));
return result;
@ -73,7 +72,7 @@ bool ODBCBridgeHelper::checkODBCBridgeIsRunning() const
{
try
{
ReadWriteBufferFromHTTP buf(ping_url, ODBCBridgeHelper::PING_METHOD, nullptr);
ReadWriteBufferFromHTTP buf(ping_url, Poco::Net::HTTPRequest::HTTP_GET, nullptr);
return checkString(ODBCBridgeHelper::PING_OK_ANSWER, buf);
}
catch (...)
@ -103,4 +102,30 @@ void ODBCBridgeHelper::startODBCBridgeSync() const
throw Exception("ODBCBridgeHelper: clickhouse-odbc-bridge is not responding", ErrorCodes::EXTERNAL_SERVER_IS_NOT_RESPONDING);
}
}
Poco::URI ODBCBridgeHelper::getMainURI() const
{
size_t bridge_port = config.getUInt("odbc_bridge.port", DEFAULT_PORT);
std::string bridge_host = config.getString("odbc_bridge.host", DEFAULT_HOST);
Poco::URI main_uri;
main_uri.setHost(bridge_host);
main_uri.setPort(bridge_port);
main_uri.setScheme("http");
main_uri.setPath(MAIN_HANDLER);
return main_uri;
}
Poco::URI ODBCBridgeHelper::getColumnsInfoURI() const
{
size_t bridge_port = config.getUInt("odbc_bridge.port", DEFAULT_PORT);
std::string bridge_host = config.getString("odbc_bridge.host", DEFAULT_HOST);
Poco::URI columns_info_uri;
columns_info_uri.setHost(bridge_host);
columns_info_uri.setPort(bridge_port);
columns_info_uri.setScheme("http");
columns_info_uri.setPath(COL_INFO_HANDLER);
return columns_info_uri;
}
}

19
dbms/src/Common/ODBCBridgeHelper.h
View File

@ -2,8 +2,8 @@
#include <Interpreters/Context.h>
#include <Poco/Logger.h>
#include <Poco/Net/HTTPRequest.h>
#include <Poco/URI.h>
#include <Poco/Util/AbstractConfiguration.h>
namespace DB
{
@ -16,7 +16,11 @@ namespace ErrorCodes
class ODBCBridgeHelper
{
private:
const Context & context_global;
using Configuration = Poco::Util::AbstractConfiguration;
const Configuration & config;
Poco::Timespan http_timeout;
std::string connection_string;
@ -31,17 +35,18 @@ public:
static constexpr inline auto DEFAULT_FORMAT = "RowBinary";
static constexpr inline auto PING_HANDLER = "/ping";
static constexpr inline auto MAIN_HANDLER = "/";
static constexpr inline auto COL_INFO_HANDLER = "/columns_info";
static constexpr inline auto PING_OK_ANSWER = "Ok.";
static const inline std::string PING_METHOD = Poco::Net::HTTPRequest::HTTP_GET;
static const inline std::string MAIN_METHOD = Poco::Net::HTTPRequest::HTTP_POST;
ODBCBridgeHelper(const Configuration & config_, const Poco::Timespan & http_timeout_, const std::string & connection_string_);
ODBCBridgeHelper(const Context & context_global_, const std::string & connection_string_);
std::vector<std::pair<std::string, std::string>> getURLParams(const NamesAndTypesList & cols, size_t max_block_size) const;
std::vector<std::pair<std::string, std::string>> getURLParams(const std::string & cols, size_t max_block_size) const;
bool checkODBCBridgeIsRunning() const;
void startODBCBridge() const;
void startODBCBridgeSync() const;
Poco::URI getMainURI() const;
Poco::URI getColumnsInfoURI() const;
};
}

101
dbms/src/Common/ProfileEvents.cpp
View File

@ -1,5 +1,7 @@
#include <Common/ProfileEvents.h>
#include <Common/CurrentThread.h>
#include <Common/typeid_cast.h>
#include <Columns/ColumnArray.h>
/// Available events. Add something here as you wish.
#define APPLY_FOR_EVENTS(M) \
@ -37,6 +39,11 @@
M(CreatedReadBufferAIO) \
M(CreatedWriteBufferOrdinary) \
M(CreatedWriteBufferAIO) \
M(DiskReadElapsedMicroseconds) \
M(DiskWriteElapsedMicroseconds) \
M(NetworkReceiveElapsedMicroseconds) \
M(NetworkSendElapsedMicroseconds) \
M(ThrottlerSleepMicroseconds) \
\
M(ReplicatedPartFetches) \
M(ReplicatedPartFailedFetches) \
@ -143,31 +150,93 @@
M(RWLockAcquiredWriteLocks) \
M(RWLockReadersWaitMilliseconds) \
M(RWLockWritersWaitMilliseconds) \
M(NetworkErrors) \
\
M(NetworkErrors)
M(RealTimeMicroseconds) \
M(UserTimeMicroseconds) \
M(SystemTimeMicroseconds) \
M(SoftPageFaults) \
M(HardPageFaults) \
M(VoluntaryContextSwitches) \
M(InvoluntaryContextSwitches) \
\
M(OSIOWaitMicroseconds) \
M(OSCPUWaitMicroseconds) \
M(OSCPUVirtualTimeMicroseconds) \
M(OSReadBytes) \
M(OSWriteBytes) \
M(OSReadChars) \
M(OSWriteChars) \
namespace ProfileEvents
{
#define M(NAME) extern const Event NAME = __COUNTER__;
#define M(NAME) extern const Event NAME = __COUNTER__;
APPLY_FOR_EVENTS(M)
#undef M
constexpr Event END = __COUNTER__;
/// Global variable, initialized by zeros.
Counter global_counters_array[END] {};
/// Initialize global counters statically
Counters global_counters(global_counters_array);
const Event Counters::num_counters = END;
Counters::Counters(VariableContext level, Counters * parent)
: counters_holder(new Counter[num_counters] {}),
parent(parent),
level(level)
{
counters = counters_holder.get();
}
void Counters::resetCounters()
{
if (counters)
{
for (Event i = 0; i < num_counters; ++i)
counters[i].store(0, std::memory_order_relaxed);
}
}
void Counters::reset()
{
parent = nullptr;
resetCounters();
}
Counters Counters::getPartiallyAtomicSnapshot() const
{
Counters res(VariableContext::Snapshot, nullptr);
for (Event i = 0; i < num_counters; ++i)
res.counters[i].store(counters[i].load(std::memory_order_relaxed), std::memory_order_relaxed);
return res;
}
const char * getDescription(Event event)
{
static const char * descriptions[] =
{
#define M(NAME) #NAME,
APPLY_FOR_EVENTS(M)
#undef M
constexpr Event END = __COUNTER__;
};
std::atomic<Count> counters[END] {}; /// Global variable, initialized by zeros.
return descriptions[event];
}
const char * getDescription(Event event)
{
static const char * descriptions[] =
{
#define M(NAME) #NAME,
APPLY_FOR_EVENTS(M)
#undef M
};
return descriptions[event];
}
Event end() { return END; }
void increment(Event event, Count amount)
{
DB::CurrentThread::getProfileEvents().increment(event, amount);
}
Event end() { return END; }
}
#undef APPLY_FOR_EVENTS

84
dbms/src/Common/ProfileEvents.h
View File

@ -1,8 +1,9 @@
#pragma once
#include <stddef.h>
#include <Common/VariableContext.h>
#include <atomic>
#include <memory>
#include <stddef.h>
/** Implements global counters for various events happening in the application
* - for high level profiling.
@ -14,19 +15,80 @@ namespace ProfileEvents
/// Event identifier (index in array).
using Event = size_t;
using Count = size_t;
using Counter = std::atomic<Count>;
class Counters;
/// Counters - how many times each event happened
extern Counters global_counters;
class Counters
{
Counter * counters = nullptr;
std::unique_ptr<Counter[]> counters_holder;
/// Used to propagate increments
Counters * parent = nullptr;
public:
VariableContext level = VariableContext::Thread;
/// By default, any instance have to increment global counters
Counters(VariableContext level = VariableContext::Thread, Counters * parent = &global_counters);
/// Global level static initializer
Counters(Counter * allocated_counters)
: counters(allocated_counters), parent(nullptr), level(VariableContext::Global) {}
Counter & operator[] (Event event)
{
return counters[event];
}
const Counter & operator[] (Event event) const
{
return counters[event];
}
inline void increment(Event event, Count amount = 1)
{
Counters * current = this;
do
{
current->counters[event].fetch_add(amount, std::memory_order_relaxed);
current = current->parent;
} while (current != nullptr);
}
/// Every single value is fetched atomically, but not all values as a whole.
Counters getPartiallyAtomicSnapshot() const;
/// Reset all counters to zero and reset parent.
void reset();
/// Get parent (thread unsafe)
Counters * getParent()
{
return parent;
}
/// Set parent (thread unsafe)
void setParent(Counters * parent_)
{
parent = parent_;
}
/// Set all counters to zero
void resetCounters();
static const Event num_counters;
};
/// Increment a counter for event. Thread-safe.
void increment(Event event, Count amount = 1);
/// Get text description of event by identifier. Returns statically allocated string.
const char * getDescription(Event event);
/// Counters - how many times each event happened.
extern std::atomic<Count> counters[];
/// Increment a counter for event. Thread-safe.
inline void increment(Event event, Count amount = 1)
{
counters[event].fetch_add(amount, std::memory_order_relaxed);
}
/// Get index just after last event identifier.
Event end();
}

2
dbms/src/Common/RWLockFIFO.cpp
View File

@ -86,7 +86,7 @@ RWLockFIFO::LockHandler RWLockFIFO::getLock(RWLockFIFO::Type type, RWLockFIFO::C
handler_ptr->it_client->info += "; " + client.info;
return handler_ptr;
return handler_ptr;
}
if (type == Type::Write || queue.empty() || queue.back().type == Type::Write)

79
dbms/src/Common/SimpleActionBlocker.h Normal file
View File

@ -0,0 +1,79 @@
#pragma once
#include <atomic>
namespace DB
{
class SimpleActionLock;
/// Similar to ActionBlocker, but without weak_ptr magic
class SimpleActionBlocker
{
using Counter = std::atomic<int>;
Counter counter = 0;
public:
SimpleActionBlocker() = default;
bool isCancelled() const { return counter > 0; }
/// Temporarily blocks corresponding actions (while the returned object is alive)
friend class SimpleActionLock;
inline SimpleActionLock cancel();
/// Cancel the actions forever.
void cancelForever() { ++counter; }
};
/// Blocks related action while a SimpleActionLock instance exists
class SimpleActionLock
{
SimpleActionBlocker * block = nullptr;
public:
SimpleActionLock() = default;
explicit SimpleActionLock(SimpleActionBlocker & block_) : block(&block_)
{
++block->counter;
}
SimpleActionLock(const SimpleActionLock &) = delete;
SimpleActionLock(SimpleActionLock && rhs) noexcept
{
*this = std::move(rhs);
}
SimpleActionLock & operator=(const SimpleActionLock &) = delete;
SimpleActionLock & operator=(SimpleActionLock && rhs) noexcept
{
if (block)
--block->counter;
block = rhs.block;
rhs.block = nullptr;
return *this;
}
~SimpleActionLock()
{
if (block)
--block->counter;
}
};
SimpleActionLock SimpleActionBlocker::cancel()
{
return SimpleActionLock(*this);
}
}

2
dbms/src/Common/StatusFile.cpp
View File

@ -30,7 +30,7 @@ StatusFile::StatusFile(const std::string & path_)
std::string contents;
{
ReadBufferFromFile in(path, 1024);
LimitReadBuffer limit_in(in, 1024);
LimitReadBuffer limit_in(in, 1024, false);
readStringUntilEOF(contents, limit_in);
}

14
dbms/src/Common/Stopwatch.cpp Normal file
View File

@ -0,0 +1,14 @@
#include <sys/resource.h>
#include "Stopwatch.h"
StopwatchRUsage::Timestamp StopwatchRUsage::Timestamp::current()
{
StopwatchRUsage::Timestamp res;
::rusage rusage;
::getrusage(RUSAGE_THREAD, &rusage);
res.user_ns = rusage.ru_utime.tv_sec * 1000000000UL + rusage.ru_utime.tv_usec * 1000UL;
res.sys_ns = rusage.ru_stime.tv_sec * 1000000000UL + rusage.ru_stime.tv_usec * 1000UL;
return res;
}

64
dbms/src/Common/Stopwatch.h
View File

@ -32,9 +32,11 @@ public:
void stop() { stop_ns = nanoseconds(); is_running = false; }
void reset() { start_ns = 0; stop_ns = 0; is_running = false; }
void restart() { start(); }
UInt64 elapsed() const { return is_running ? nanoseconds() - start_ns : stop_ns - start_ns; }
UInt64 elapsedMilliseconds() const { return elapsed() / 1000000UL; }
double elapsedSeconds() const { return static_cast<double>(elapsed()) / 1000000000ULL; }
UInt64 elapsed() const { return elapsedNanoseconds(); }
UInt64 elapsedNanoseconds() const { return is_running ? nanoseconds() - start_ns : stop_ns - start_ns; }
UInt64 elapsedMicroseconds() const { return elapsedNanoseconds() / 1000U; }
UInt64 elapsedMilliseconds() const { return elapsedNanoseconds() / 1000000UL; }
double elapsedSeconds() const { return static_cast<double>(elapsedNanoseconds()) / 1000000000ULL; }
private:
UInt64 start_ns = 0;
@ -131,3 +133,59 @@ private:
/// Most significant bit is a lock. When it is set, compareAndRestartDeferred method will return false.
UInt64 nanoseconds() const { return StopWatchDetail::nanoseconds(clock_type) & 0x7FFFFFFFFFFFFFFFULL; }
};
/// Like ordinary StopWatch, but uses getrusage() system call
struct StopwatchRUsage
{
StopwatchRUsage() = default;
void start() { start_ts = Timestamp::current(); is_running = true; }
void stop() { stop_ts = Timestamp::current(); is_running = false; }
void reset() { start_ts = Timestamp(); stop_ts = Timestamp(); is_running = false; }
void restart() { start(); }
UInt64 elapsed(bool count_user = true, bool count_sys = true) const
{
return elapsedNanoseconds(count_user, count_sys);
}
UInt64 elapsedNanoseconds(bool count_user = true, bool count_sys = true) const
{
return (is_running ? Timestamp::current() : stop_ts).nanoseconds(count_user, count_sys) - start_ts.nanoseconds(count_user, count_sys);
}
UInt64 elapsedMicroseconds(bool count_user = true, bool count_sys = true) const
{
return elapsedNanoseconds(count_user, count_sys) / 1000UL;
}
UInt64 elapsedMilliseconds(bool count_user = true, bool count_sys = true) const
{
return elapsedNanoseconds(count_user, count_sys) / 1000000UL;
}
double elapsedSeconds(bool count_user = true, bool count_sys = true) const
{
return static_cast<double>(elapsedNanoseconds(count_user, count_sys)) / 1000000000.0;
}
private:
struct Timestamp
{
UInt64 user_ns = 0;
UInt64 sys_ns = 0;
static Timestamp current();
UInt64 nanoseconds(bool count_user = true, bool count_sys = true) const
{
return (count_user ? user_ns : 0) + (count_sys ? sys_ns : 0);
}
};
Timestamp start_ts;
Timestamp stop_ts;
bool is_running = false;
};

254
dbms/src/Common/TaskStatsInfoGetter.cpp Normal file
View File

@ -0,0 +1,254 @@
#include <Common/TaskStatsInfoGetter.h>
#include <Common/Exception.h>
#include <Core/Types.h>
#include <errno.h>
#include <linux/genetlink.h>
#include <linux/netlink.h>
#include <linux/taskstats.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <unistd.h>
#include <syscall.h>
/// Basic idea is motivated by "iotop" tool.
/// More info: https://www.kernel.org/doc/Documentation/accounting/taskstats.txt
#define GENLMSG_DATA(glh) ((void *)((char*)NLMSG_DATA(glh) + GENL_HDRLEN))
#define GENLMSG_PAYLOAD(glh) (NLMSG_PAYLOAD(glh, 0) - GENL_HDRLEN)
#define NLA_DATA(na) ((void *)((char*)(na) + NLA_HDRLEN))
#define NLA_PAYLOAD(len) (len - NLA_HDRLEN)
namespace DB
{
namespace ErrorCodes
{
extern const int NETLINK_ERROR;
}
namespace
{
static size_t constexpr MAX_MSG_SIZE = 1024;
struct NetlinkMessage
{
::nlmsghdr n;
::genlmsghdr g;
char buf[MAX_MSG_SIZE];
};
int sendCommand(
int sock_fd,
UInt16 nlmsg_type,
UInt32 nlmsg_pid,
UInt8 genl_cmd,
UInt16 nla_type,
void * nla_data,
int nla_len) noexcept
{
NetlinkMessage msg{};
msg.n.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN);
msg.n.nlmsg_type = nlmsg_type;
msg.n.nlmsg_flags = NLM_F_REQUEST;
msg.n.nlmsg_seq = 0;
msg.n.nlmsg_pid = nlmsg_pid;
msg.g.cmd = genl_cmd;
msg.g.version = 1;
::nlattr * attr = static_cast<::nlattr *>(GENLMSG_DATA(&msg));
attr->nla_type = nla_type;
attr->nla_len = nla_len + 1 + NLA_HDRLEN;
memcpy(NLA_DATA(attr), nla_data, nla_len);
msg.n.nlmsg_len += NLMSG_ALIGN(attr->nla_len);
char * buf = reinterpret_cast<char *>(&msg);
ssize_t buflen = msg.n.nlmsg_len;
::sockaddr_nl nladdr{};
nladdr.nl_family = AF_NETLINK;
while (true)
{
ssize_t r = ::sendto(sock_fd, buf, buflen, 0, reinterpret_cast<const ::sockaddr *>(&nladdr), sizeof(nladdr));
if (r >= buflen)
break;
if (r > 0)
{
buf += r;
buflen -= r;
}
else if (errno != EAGAIN)
return -1;
}
return 0;
}
UInt16 getFamilyId(int nl_sock_fd) noexcept
{
struct
{
::nlmsghdr header;
::genlmsghdr ge_header;
char buf[256];
} answer;
static char name[] = TASKSTATS_GENL_NAME;
if (sendCommand(
nl_sock_fd, GENL_ID_CTRL, getpid(), CTRL_CMD_GETFAMILY,
CTRL_ATTR_FAMILY_NAME, (void *) name,
strlen(TASKSTATS_GENL_NAME) + 1))
return 0;
UInt16 id = 0;
ssize_t rep_len = ::recv(nl_sock_fd, &answer, sizeof(answer), 0);
if (answer.header.nlmsg_type == NLMSG_ERROR || (rep_len < 0) || !NLMSG_OK((&answer.header), rep_len))
return 0;
const ::nlattr * attr;
attr = static_cast<const ::nlattr *>(GENLMSG_DATA(&answer));
attr = reinterpret_cast<const ::nlattr *>(reinterpret_cast<const char *>(attr) + NLA_ALIGN(attr->nla_len));
if (attr->nla_type == CTRL_ATTR_FAMILY_ID)
id = *static_cast<const UInt16 *>(NLA_DATA(attr));
return id;
}
}
TaskStatsInfoGetter::TaskStatsInfoGetter() = default;
void TaskStatsInfoGetter::init()
{
if (netlink_socket_fd >= 0)
return;
netlink_socket_fd = ::socket(PF_NETLINK, SOCK_RAW, NETLINK_GENERIC);
if (netlink_socket_fd < 0)
throwFromErrno("Can't create PF_NETLINK socket");
::sockaddr_nl addr{};
addr.nl_family = AF_NETLINK;
if (::bind(netlink_socket_fd, reinterpret_cast<const ::sockaddr *>(&addr), sizeof(addr)) < 0)
throwFromErrno("Can't bind PF_NETLINK socket");
netlink_family_id = getFamilyId(netlink_socket_fd);
}
bool TaskStatsInfoGetter::getStatImpl(int tid, ::taskstats & out_stats, bool throw_on_error)
{
init();
if (sendCommand(netlink_socket_fd, netlink_family_id, tid, TASKSTATS_CMD_GET, TASKSTATS_CMD_ATTR_PID, &tid, sizeof(pid_t)))
throwFromErrno("Can't send a Netlink command");
NetlinkMessage msg;
ssize_t rv = ::recv(netlink_socket_fd, &msg, sizeof(msg), 0);
if (msg.n.nlmsg_type == NLMSG_ERROR || !NLMSG_OK((&msg.n), rv))
{
const ::nlmsgerr * err = static_cast<const ::nlmsgerr *>(NLMSG_DATA(&msg));
if (throw_on_error)
throw Exception("Can't get Netlink response, error: " + std::to_string(err->error), ErrorCodes::NETLINK_ERROR);
else
return false;
}
rv = GENLMSG_PAYLOAD(&msg.n);
const ::nlattr * attr = static_cast<const ::nlattr *>(GENLMSG_DATA(&msg));
ssize_t len = 0;
while (len < rv)
{
len += NLA_ALIGN(attr->nla_len);
if (attr->nla_type == TASKSTATS_TYPE_AGGR_TGID || attr->nla_type == TASKSTATS_TYPE_AGGR_PID)
{
int aggr_len = NLA_PAYLOAD(attr->nla_len);
int len2 = 0;
attr = static_cast<const ::nlattr *>(NLA_DATA(attr));
while (len2 < aggr_len)
{
if (attr->nla_type == TASKSTATS_TYPE_STATS)
{
const ::taskstats * ts = static_cast<const ::taskstats *>(NLA_DATA(attr));
out_stats = *ts;
}
len2 += NLA_ALIGN(attr->nla_len);
attr = reinterpret_cast<const ::nlattr *>(reinterpret_cast<const char *>(attr) + len2);
}
}
attr = reinterpret_cast<const ::nlattr *>(reinterpret_cast<const char *>(GENLMSG_DATA(&msg)) + len);
}
return true;
}
void TaskStatsInfoGetter::getStat(::taskstats & stat, int tid)
{
tid = tid < 0 ? getDefaultTID() : tid;
getStatImpl(tid, stat, true);
}
bool TaskStatsInfoGetter::tryGetStat(::taskstats & stat, int tid)
{
tid = tid < 0 ? getDefaultTID() : tid;
return getStatImpl(tid, stat, false);
}
TaskStatsInfoGetter::~TaskStatsInfoGetter()
{
if (netlink_socket_fd >= 0)
close(netlink_socket_fd);
}
int TaskStatsInfoGetter::getCurrentTID()
{
/// This call is always successful. - man gettid
return static_cast<int>(syscall(SYS_gettid));
}
int TaskStatsInfoGetter::getDefaultTID()
{
if (default_tid < 0)
default_tid = getCurrentTID();
return default_tid;
}
static bool tryGetTaskStats()
{
TaskStatsInfoGetter getter;
::taskstats stat;
return getter.tryGetStat(stat);
}
bool TaskStatsInfoGetter::checkProcessHasRequiredPermissions()
{
/// It is thread- and exception- safe since C++11
static bool res = tryGetTaskStats();
return res;
}
}

43
dbms/src/Common/TaskStatsInfoGetter.h Normal file
View File

@ -0,0 +1,43 @@
#pragma once
#include <Core/Types.h>
struct taskstats;
namespace DB
{
class Exception;
/// Get taskstat info from OS kernel via Netlink protocol.
class TaskStatsInfoGetter
{
public:
TaskStatsInfoGetter();
TaskStatsInfoGetter(const TaskStatsInfoGetter &) = delete;
void getStat(::taskstats & stat, int tid = -1);
bool tryGetStat(::taskstats & stat, int tid = -1);
~TaskStatsInfoGetter();
/// Make a syscall and returns Linux thread id
static int getCurrentTID();
/// Whether the current process has permissions (sudo or cap_net_admin capabilties) to get taskstats info
static bool checkProcessHasRequiredPermissions();
private:
/// Caches current thread tid to avoid extra sys calls
int getDefaultTID();
int default_tid = -1;
bool getStatImpl(int tid, ::taskstats & out_stats, bool throw_on_error = false);
void init();
int netlink_socket_fd = -1;
UInt16 netlink_family_id = 0;
};
}

144
dbms/src/Common/ThreadProfileEvents.h Normal file
View File

@ -0,0 +1,144 @@
#pragma once
#include <Common/TaskStatsInfoGetter.h>
#include <Common/ProfileEvents.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <pthread.h>
#include <linux/taskstats.h>
namespace ProfileEvents
{
extern const Event RealTimeMicroseconds;
extern const Event UserTimeMicroseconds;
extern const Event SystemTimeMicroseconds;
extern const Event SoftPageFaults;
extern const Event HardPageFaults;
extern const Event VoluntaryContextSwitches;
extern const Event InvoluntaryContextSwitches;
extern const Event OSIOWaitMicroseconds;
extern const Event OSCPUWaitMicroseconds;
extern const Event OSCPUVirtualTimeMicroseconds;
extern const Event OSReadChars;
extern const Event OSWriteChars;
extern const Event OSReadBytes;
extern const Event OSWriteBytes;
}
namespace DB
{
/// Handles overflow
template <typename TUInt>
inline TUInt safeDiff(TUInt prev, TUInt curr)
{
return curr >= prev ? curr - prev : 0;
}
inline UInt64 getCurrentTimeNanoseconds(clockid_t clock_type = CLOCK_MONOTONIC)
{
struct timespec ts;
clock_gettime(clock_type, &ts);
return ts.tv_sec * 1000000000ULL + ts.tv_nsec;
}
struct RUsageCounters
{
/// In nanoseconds
UInt64 real_time = 0;
UInt64 user_time = 0;
UInt64 sys_time = 0;
UInt64 soft_page_faults = 0;
UInt64 hard_page_faults = 0;
RUsageCounters() = default;
RUsageCounters(const ::rusage & rusage_, UInt64 real_time_)
{
set(rusage_, real_time_);
}
void set(const ::rusage & rusage, UInt64 real_time_)
{
real_time = real_time_;
user_time = rusage.ru_utime.tv_sec * 1000000000UL + rusage.ru_utime.tv_usec * 1000UL;
sys_time = rusage.ru_stime.tv_sec * 1000000000UL + rusage.ru_stime.tv_usec * 1000UL;
soft_page_faults = static_cast<UInt64>(rusage.ru_minflt);
hard_page_faults = static_cast<UInt64>(rusage.ru_majflt);
}
static RUsageCounters zeros(UInt64 real_time_ = getCurrentTimeNanoseconds())
{
RUsageCounters res;
res.real_time = real_time_;
return res;
}
static RUsageCounters current(UInt64 real_time_ = getCurrentTimeNanoseconds())
{
::rusage rusage;
::getrusage(RUSAGE_THREAD, &rusage);
return RUsageCounters(rusage, real_time_);
}
static void incrementProfileEvents(const RUsageCounters & prev, const RUsageCounters & curr, ProfileEvents::Counters & profile_events)
{
profile_events.increment(ProfileEvents::RealTimeMicroseconds, (curr.real_time - prev.real_time) / 1000U);
profile_events.increment(ProfileEvents::UserTimeMicroseconds, (curr.user_time - prev.user_time) / 1000U);
profile_events.increment(ProfileEvents::SystemTimeMicroseconds, (curr.sys_time - prev.sys_time) / 1000U);
profile_events.increment(ProfileEvents::SoftPageFaults, curr.soft_page_faults - prev.soft_page_faults);
profile_events.increment(ProfileEvents::HardPageFaults, curr.hard_page_faults - prev.hard_page_faults);
}
static void updateProfileEvents(RUsageCounters & last_counters, ProfileEvents::Counters & profile_events)
{
auto current_counters = current();
incrementProfileEvents(last_counters, current_counters, profile_events);
last_counters = current_counters;
}
};
struct TasksStatsCounters
{
::taskstats stat;
TasksStatsCounters() = default;
static TasksStatsCounters current();
static void incrementProfileEvents(const TasksStatsCounters & prev, const TasksStatsCounters & curr, ProfileEvents::Counters & profile_events)
{
profile_events.increment(ProfileEvents::OSCPUWaitMicroseconds,
safeDiff(prev.stat.cpu_delay_total, curr.stat.cpu_delay_total) / 1000U);
profile_events.increment(ProfileEvents::OSIOWaitMicroseconds,
safeDiff(prev.stat.blkio_delay_total, curr.stat.blkio_delay_total) / 1000U);
profile_events.increment(ProfileEvents::OSCPUVirtualTimeMicroseconds,
safeDiff(prev.stat.cpu_run_virtual_total, curr.stat.cpu_run_virtual_total) / 1000U);
/// Too old struct version, do not read new fields
if (curr.stat.version < TASKSTATS_VERSION)
return;
profile_events.increment(ProfileEvents::OSReadChars, safeDiff(prev.stat.read_char, curr.stat.read_char));
profile_events.increment(ProfileEvents::OSWriteChars, safeDiff(prev.stat.write_char, curr.stat.write_char));
profile_events.increment(ProfileEvents::OSReadBytes, safeDiff(prev.stat.read_bytes, curr.stat.read_bytes));
profile_events.increment(ProfileEvents::OSWriteBytes, safeDiff(prev.stat.write_bytes, curr.stat.write_bytes));
}
static void updateProfileEvents(TasksStatsCounters & last_counters, ProfileEvents::Counters & profile_events)
{
auto current_counters = current();
incrementProfileEvents(last_counters, current_counters, profile_events);
last_counters = current_counters;
}
};
}

118
dbms/src/Common/ThreadStatus.cpp Normal file
View File

@ -0,0 +1,118 @@
#include "ThreadStatus.h"
#include <common/logger_useful.h>
#include <Common/CurrentThread.h>
#include <Common/Exception.h>
#include <Common/ThreadProfileEvents.h>
#include <Poco/Thread.h>
#include <Poco/Ext/ThreadNumber.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int PTHREAD_ERROR;
}
/// Order of current_thread and current_thread_scope matters
thread_local ThreadStatusPtr current_thread = ThreadStatus::create();
thread_local ThreadStatus::CurrentThreadScope current_thread_scope;
TasksStatsCounters TasksStatsCounters::current()
{
TasksStatsCounters res;
current_thread->taskstats_getter->getStat(res.stat, current_thread->os_thread_id);
return res;
}
ThreadStatus::ThreadStatus()
{
thread_number = Poco::ThreadNumber::get();
os_thread_id = TaskStatsInfoGetter::getCurrentTID();
last_rusage = std::make_unique<RUsageCounters>();
last_taskstats = std::make_unique<TasksStatsCounters>();
taskstats_getter = std::make_unique<TaskStatsInfoGetter>();
memory_tracker.setDescription("(for thread)");
log = &Poco::Logger::get("ThreadStatus");
/// NOTE: It is important not to do any non-trivial actions (like updating ProfileEvents or logging) before ThreadStatus is created
/// Otherwise it could lead to SIGSEGV due to current_thread dereferencing
}
ThreadStatusPtr ThreadStatus::create()
{
return ThreadStatusPtr(new ThreadStatus);
}
ThreadStatus::~ThreadStatus() = default;
void ThreadStatus::initPerformanceCounters()
{
performance_counters_finalized = false;
/// Clear stats from previous query if a new query is started
/// TODO: make separate query_thread_performance_counters and thread_performance_counters
performance_counters.resetCounters();
memory_tracker.resetCounters();
memory_tracker.setDescription("(for thread)");
query_start_time_nanoseconds = getCurrentTimeNanoseconds();
query_start_time = time(nullptr);
++queries_started;
*last_rusage = RUsageCounters::current(query_start_time_nanoseconds);
has_permissions_for_taskstats = TaskStatsInfoGetter::checkProcessHasRequiredPermissions();
if (has_permissions_for_taskstats)
*last_taskstats = TasksStatsCounters::current();
}
void ThreadStatus::updatePerformanceCounters()
{
try
{
RUsageCounters::updateProfileEvents(*last_rusage, performance_counters);
if (has_permissions_for_taskstats)
TasksStatsCounters::updateProfileEvents(*last_taskstats, performance_counters);
}
catch (...)
{
tryLogCurrentException(log);
}
}
void ThreadStatus::assertState(const std::initializer_list<int> & permitted_states, const char * description)
{
for (auto permitted_state : permitted_states)
{
if (getCurrentState() == permitted_state)
return;
}
std::stringstream ss;
ss << "Unexpected thread state " << getCurrentState();
if (description)
ss << ": " << description;
throw Exception(ss.str(), ErrorCodes::LOGICAL_ERROR);
}
void ThreadStatus::attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue)
{
logs_queue_ptr = logs_queue;
if (!thread_group)
return;
std::unique_lock lock(thread_group->mutex);
thread_group->logs_queue_ptr = logs_queue;
}
}

197
dbms/src/Common/ThreadStatus.h Normal file
View File

@ -0,0 +1,197 @@
#pragma once
#include <Common/ProfileEvents.h>
#include <Common/MemoryTracker.h>
#include <IO/Progress.h>
#include <memory>
#include <map>
#include <mutex>
#include <shared_mutex>
namespace Poco
{
class Logger;
}
namespace DB
{
class Context;
class QueryStatus;
class ThreadStatus;
using ThreadStatusPtr = std::shared_ptr<ThreadStatus>;
class QueryThreadLog;
struct TasksStatsCounters;
struct RUsageCounters;
class TaskStatsInfoGetter;
class InternalTextLogsQueue;
using InternalTextLogsQueuePtr = std::shared_ptr<InternalTextLogsQueue>;
using InternalTextLogsQueueWeakPtr = std::weak_ptr<InternalTextLogsQueue>;
class ThreadGroupStatus
{
public:
mutable std::shared_mutex mutex;
ProfileEvents::Counters performance_counters{VariableContext::Process};
MemoryTracker memory_tracker{VariableContext::Process};
Context * query_context = nullptr;
Context * global_context = nullptr;
InternalTextLogsQueueWeakPtr logs_queue_ptr;
/// Key is Poco's thread_id
using QueryThreadStatuses = std::map<UInt32, ThreadStatusPtr>;
QueryThreadStatuses thread_statuses;
/// The first thread created this thread group
ThreadStatusPtr master_thread;
String query;
};
using ThreadGroupStatusPtr = std::shared_ptr<ThreadGroupStatus>;
class ThreadStatus : public std::enable_shared_from_this<ThreadStatus>
{
public:
/// Poco's thread number (the same number is used in logs)
UInt32 thread_number = 0;
/// Linux's PID (or TGID) (the same id is shown by ps util)
Int32 os_thread_id = -1;
/// TODO: merge them into common entity
ProfileEvents::Counters performance_counters{VariableContext::Thread};
MemoryTracker memory_tracker{VariableContext::Thread};
/// Statistics of read and write rows/bytes
Progress progress_in;
Progress progress_out;
public:
static ThreadStatusPtr create();
ThreadGroupStatusPtr getThreadGroup() const
{
return thread_group;
}
enum ThreadState
{
DetachedFromQuery = 0, /// We just created thread or it is a background thread
AttachedToQuery, /// Thread executes enqueued query
Died, /// Thread does not exist
};
int getCurrentState() const
{
return thread_state.load(std::memory_order_relaxed);
}
String getQueryID();
/// Starts new query and create new thread group for it, current thread becomes master thread of the query
void initializeQuery();
/// Attaches slave thread to existing thread group
void attachQuery(const ThreadGroupStatusPtr & thread_group_, bool check_detached = true);
InternalTextLogsQueuePtr getInternalTextLogsQueue() const
{
return thread_state == Died ? nullptr : logs_queue_ptr.lock();
}
void attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue);
/// Sets query context for current thread and its thread group
/// NOTE: query_context have to be alive until detachQuery() is called
void attachQueryContext(Context & query_context);
/// Update several ProfileEvents counters
void updatePerformanceCounters();
/// Update ProfileEvents and dumps info to system.query_thread_log
void finalizePerformanceCounters();
/// Detaches thread from the thread group and the query, dumps performance counters if they have not been dumped
void detachQuery(bool exit_if_already_detached = false, bool thread_exits = false);
~ThreadStatus();
protected:
ThreadStatus();
void initPerformanceCounters();
void logToQueryThreadLog(QueryThreadLog & thread_log);
void assertState(const std::initializer_list<int> & permitted_states, const char * description = nullptr);
ThreadGroupStatusPtr thread_group;
std::atomic<int> thread_state{ThreadState::DetachedFromQuery};
/// Is set once
Context * global_context = nullptr;
/// Use it only from current thread
Context * query_context = nullptr;
/// A logs queue used by TCPHandler to pass logs to a client
InternalTextLogsQueueWeakPtr logs_queue_ptr;
bool performance_counters_finalized = false;
UInt64 query_start_time_nanoseconds = 0;
time_t query_start_time = 0;
size_t queries_started = 0;
Poco::Logger * log = nullptr;
friend class CurrentThread;
friend struct TasksStatsCounters;
/// Use ptr not to add extra dependencies in the header
std::unique_ptr<RUsageCounters> last_rusage;
std::unique_ptr<TasksStatsCounters> last_taskstats;
std::unique_ptr<TaskStatsInfoGetter> taskstats_getter;
bool has_permissions_for_taskstats = false;
public:
/// Implicitly finalizes current thread in the destructor
class CurrentThreadScope
{
public:
void (*deleter)() = nullptr;
CurrentThreadScope() = default;
~CurrentThreadScope()
{
try
{
if (deleter)
deleter();
}
catch (...)
{
std::terminate();
}
}
};
private:
static void defaultThreadDeleter();
};
extern thread_local ThreadStatusPtr current_thread;
extern thread_local ThreadStatus::CurrentThreadScope current_thread_scope;
}

15
dbms/src/Common/Throttler.h
View File

@ -5,10 +5,17 @@
#include <memory>
#include <Common/Stopwatch.h>
#include <Common/Exception.h>
#include <Common/ProfileEvents.h>
#include <IO/WriteHelpers.h>
#include <port/clock.h>
namespace ProfileEvents
{
extern const Event ThrottlerSleepMicroseconds;
}
namespace DB
{
@ -69,10 +76,14 @@ public:
if (desired_ns > elapsed_ns)
{
UInt64 sleep_ns = desired_ns - elapsed_ns;
timespec sleep_ts;
::timespec sleep_ts;
sleep_ts.tv_sec = sleep_ns / 1000000000;
sleep_ts.tv_nsec = sleep_ns % 1000000000;
nanosleep(&sleep_ts, nullptr); /// NOTE Returns early in case of a signal. This is considered normal.
/// NOTE: Returns early in case of a signal. This is considered normal.
::nanosleep(&sleep_ts, nullptr);
ProfileEvents::increment(ProfileEvents::ThrottlerSleepMicroseconds, sleep_ns / 1000UL);
}
}

10
dbms/src/Common/UInt128.h
View File

@ -28,9 +28,17 @@ struct UInt128
UInt64 high;
UInt128() = default;
explicit UInt128(const UInt64 rhs) : low(rhs), high() {}
explicit UInt128(const UInt64 low, const UInt64 high) : low(low), high(high) {}
#if 1
explicit UInt128(const unsigned __int128 rhs)
: low(rhs & 0xffffffffffffffffll),
high(rhs >> 64)
{}
#else
explicit UInt128(const UInt64 rhs) : low(rhs), high() {}
#endif
auto tuple() const { return std::tie(high, low); }
bool inline operator== (const UInt128 rhs) const { return tuple() == rhs.tuple(); }

12
dbms/src/Common/VariableContext.h Normal file
View File

@ -0,0 +1,12 @@
#pragma once
/// Used in ProfileEvents and MemoryTracker to determine their hierarchy level
/// The less value the higher level (zero level is the root)
enum class VariableContext
{
Global = 0,
User, /// Group of processes
Process, /// For example, a query or a merge
Thread, /// A thread of a process
Snapshot /// Does not belong to anybody
};

28
dbms/src/Common/setThreadName.cpp
View File

@ -6,10 +6,20 @@
#else
#include <sys/prctl.h>
#endif
#include <pthread.h>
#include <cstring>
#include <Common/Exception.h>
#include <Common/setThreadName.h>
namespace DB
{
namespace ErrorCodes
{
extern const int PTHREAD_ERROR;
}
}
void setThreadName(const char * name)
{
@ -22,5 +32,21 @@ void setThreadName(const char * name)
#else
if (0 != prctl(PR_SET_NAME, name, 0, 0, 0))
#endif
DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME...)");
DB::throwFromErrno("Cannot set thread name with prctl(PR_SET_NAME, ...)");
}
std::string getThreadName()
{
std::string name(16, '\0');
#if defined(__FreeBSD__) || defined(__APPLE__)
if (pthread_get_name_np(pthread_self(), name.data(), name.size());
throw DB::Exception("Cannot get thread name with pthread_get_name_np()", DB::ErrorCodes::PTHREAD_ERROR);
#else
if (0 != prctl(PR_GET_NAME, name.data(), 0, 0, 0))
#endif
DB::throwFromErrno("Cannot get thread name with prctl(PR_GET_NAME)");
name.resize(std::strlen(name.data()));
return name;
}

3
dbms/src/Common/setThreadName.h
View File

@ -1,7 +1,10 @@
#pragma once
#include <string>
/** Sets the thread name (maximum length is 15 bytes),
* which will be visible in ps, gdb, /proc,
* for convenience of observation and debugging.
*/
void setThreadName(const char * name);
std::string getThreadName();

3
dbms/src/Common/tests/CMakeLists.txt
View File

@ -71,6 +71,3 @@ target_link_libraries (cow_columns clickhouse_common_io)
add_executable (stopwatch stopwatch.cpp)
target_link_libraries (stopwatch clickhouse_common_io)
add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp)
target_link_libraries (validate-odbc-connection-string dbms)

2
dbms/src/Common/tests/gtest_rw_lock_fifo.cpp.cpp
View File

@ -32,7 +32,7 @@ TEST(Common, RWLockFIFO_1)
auto func = [&] (size_t threads, int round)
{
for (int i = 0; i < cycles; ++i)
for (int i = 0; i < cycles; ++i)
{
auto type = (std::uniform_int_distribution<>(0, 9)(gen) >= round) ? RWLockFIFO::Read : RWLockFIFO::Write;
auto sleep_for = std::chrono::duration<int, std::micro>(std::uniform_int_distribution<>(1, 100)(gen));

30
dbms/src/Core/AccurateComparison.h
View File

@ -332,10 +332,32 @@ inline bool equalsOp<DB::Float32, DB::UInt128>(DB::Float32 f, DB::UInt128 u)
return equalsOp(static_cast<DB::Float64>(f), u);
}
inline bool greaterOp(DB::Int128 i, DB::Float64 f) { return static_cast<DB::Int128>(f) < i; }
inline bool greaterOp(DB::Int128 i, DB::Float32 f) { return static_cast<DB::Int128>(f) < i; }
inline bool greaterOp(DB::Float64 f, DB::Int128 i) { return static_cast<DB::Int128>(f) > i; }
inline bool greaterOp(DB::Float32 f, DB::Int128 i) { return static_cast<DB::Int128>(f) > i; }
inline bool greaterOp(DB::Int128 i, DB::Float64 f)
{
static constexpr __int128 min_int128 = __int128(0x8000000000000000ll) << 64;
static constexpr __int128 max_int128 = (__int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll;
if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR)
return static_cast<DB::Float64>(i) > f;
return (f < static_cast<DB::Float64>(min_int128))
|| (f < static_cast<DB::Float64>(max_int128) && i > static_cast<DB::Int128>(f));
}
inline bool greaterOp(DB::Float64 f, DB::Int128 i)
{
static constexpr __int128 min_int128 = __int128(0x8000000000000000ll) << 64;
static constexpr __int128 max_int128 = (__int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll;
if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR)
return f > static_cast<DB::Float64>(i);
return (f >= static_cast<DB::Float64>(max_int128))
|| (f > static_cast<DB::Float64>(min_int128) && static_cast<DB::Int128>(f) > i);
}
inline bool greaterOp(DB::Int128 i, DB::Float32 f) { return greaterOp(i, static_cast<DB::Float64>(f)); }
inline bool greaterOp(DB::Float32 f, DB::Int128 i) { return greaterOp(static_cast<DB::Float64>(f), i); }
inline bool equalsOp(DB::Int128 i, DB::Float64 f) { return i == static_cast<DB::Int128>(f) && static_cast<DB::Float64>(i) == f; }
inline bool equalsOp(DB::Int128 i, DB::Float32 f) { return i == static_cast<DB::Int128>(f) && static_cast<DB::Float32>(i) == f; }

1
dbms/src/Core/Defines.h
View File

@ -46,6 +46,7 @@
#define DBMS_MIN_REVISION_WITH_TIME_ZONE_PARAMETER_IN_DATETIME_DATA_TYPE 54337
#define DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME 54372
#define DBMS_MIN_REVISION_WITH_VERSION_PATCH 54401
#define DBMS_MIN_REVISION_WITH_SERVER_LOGS 54406
/// Version of ClickHouse TCP protocol. Set to git tag with latest protocol change.
#define DBMS_TCP_PROTOCOL_VERSION 54226

182
dbms/src/Core/ExternalTable.cpp Normal file
View File

@ -0,0 +1,182 @@
#include <boost/program_options.hpp>
#include <boost/algorithm/string.hpp>
#include <DataStreams/AsynchronousBlockInputStream.h>
#include <DataTypes/DataTypeFactory.h>
#include <Interpreters/Context.h>
#include <IO/copyData.h>
#include <IO/ReadBufferFromIStream.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/LimitReadBuffer.h>
#include <Storages/StorageMemory.h>
#include <Poco/Net/HTMLForm.h>
#include <Poco/Net/MessageHeader.h>
#include <Common/HTMLForm.h>
#include <Core/ExternalTable.h>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
ExternalTableData BaseExternalTable::getData(const Context & context)
{
initReadBuffer();
initSampleBlock();
auto input = context.getInputFormat(format, *read_buffer, sample_block, DEFAULT_BLOCK_SIZE);
return std::make_pair(std::make_shared<AsynchronousBlockInputStream>(input), name);
}
void BaseExternalTable::clean()
{
name = "";
file = "";
format = "";
structure.clear();
sample_block = Block();
read_buffer.reset();
}
/// Function for debugging information output
void BaseExternalTable::write()
{
std::cerr << "file " << file << std::endl;
std::cerr << "name " << name << std::endl;
std::cerr << "format " << format << std::endl;
std::cerr << "structure: \n";
for (size_t i = 0; i < structure.size(); ++i)
std::cerr << "\t" << structure[i].first << " " << structure[i].second << std::endl;
}
std::vector<std::string> BaseExternalTable::split(const std::string & s, const std::string & d)
{
std::vector<std::string> res;
boost::split(res, s, boost::algorithm::is_any_of(d), boost::algorithm::token_compress_on);
return res;
}
void BaseExternalTable::parseStructureFromStructureField(const std::string & argument)
{
std::vector<std::string> vals = split(argument, " ,");
if (vals.size() & 1)
throw Exception("Odd number of attributes in section structure", ErrorCodes::BAD_ARGUMENTS);
for (size_t i = 0; i < vals.size(); i += 2)
structure.emplace_back(vals[i], vals[i + 1]);
}
void BaseExternalTable::parseStructureFromTypesField(const std::string & argument)
{
std::vector<std::string> vals = split(argument, " ,");
for (size_t i = 0; i < vals.size(); ++i)
structure.emplace_back("_" + toString(i + 1), vals[i]);
}
void BaseExternalTable::initSampleBlock()
{
const DataTypeFactory & data_type_factory = DataTypeFactory::instance();
for (size_t i = 0; i < structure.size(); ++i)
{
ColumnWithTypeAndName column;
column.name = structure[i].first;
column.type = data_type_factory.get(structure[i].second);
column.column = column.type->createColumn();
sample_block.insert(std::move(column));
}
}
void ExternalTable::initReadBuffer()
{
if (file == "-")
read_buffer = std::make_unique<ReadBufferFromFileDescriptor>(STDIN_FILENO);
else
read_buffer = std::make_unique<ReadBufferFromFile>(file);
}
ExternalTable::ExternalTable(const boost::program_options::variables_map & external_options)
{
if (external_options.count("file"))
file = external_options["file"].as<std::string>();
else
throw Exception("--file field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
if (external_options.count("name"))
name = external_options["name"].as<std::string>();
else
throw Exception("--name field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
if (external_options.count("format"))
format = external_options["format"].as<std::string>();
else
throw Exception("--format field have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
if (external_options.count("structure"))
parseStructureFromStructureField(external_options["structure"].as<std::string>());
else if (external_options.count("types"))
parseStructureFromTypesField(external_options["types"].as<std::string>());
else
throw Exception("Neither --structure nor --types have not been provided for external table", ErrorCodes::BAD_ARGUMENTS);
}
void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header, std::istream & stream)
{
const Settings & settings = context.getSettingsRef();
/// The buffer is initialized here, not in the virtual function initReadBuffer
read_buffer_impl = std::make_unique<ReadBufferFromIStream>(stream);
if (settings.http_max_multipart_form_data_size)
read_buffer = std::make_unique<LimitReadBuffer>(
*read_buffer_impl, settings.http_max_multipart_form_data_size,
true, "the maximum size of multipart/form-data. This limit can be tuned by 'http_max_multipart_form_data_size' setting");
else
read_buffer = std::move(read_buffer_impl);
/// Retrieve a collection of parameters from MessageHeader
Poco::Net::NameValueCollection content;
std::string label;
Poco::Net::MessageHeader::splitParameters(header.get("Content-Disposition"), label, content);
/// Get parameters
name = content.get("name", "_data");
format = params.get(name + "_format", "TabSeparated");
if (params.has(name + "_structure"))
parseStructureFromStructureField(params.get(name + "_structure"));
else if (params.has(name + "_types"))
parseStructureFromTypesField(params.get(name + "_types"));
else
throw Exception("Neither structure nor types have not been provided for external table " + name + ". Use fields " + name + "_structure or " + name + "_types to do so.", ErrorCodes::BAD_ARGUMENTS);
ExternalTableData data = getData(context);
/// Create table
NamesAndTypesList columns = sample_block.getNamesAndTypesList();
StoragePtr storage = StorageMemory::create(data.second, ColumnsDescription{columns});
storage->startup();
context.addExternalTable(data.second, storage);
BlockOutputStreamPtr output = storage->write(ASTPtr(), settings);
/// Write data
data.first->readPrefix();
output->writePrefix();
while(Block block = data.first->read())
output->write(block);
data.first->readSuffix();
output->writeSuffix();
/// We are ready to receive the next file, for this we clear all the information received
clean();
}
}

111
dbms/src/Core/ExternalTable.h Normal file
View File

@ -0,0 +1,111 @@
#pragma once
#include <string>
#include <vector>
#include <memory>
#include <iosfwd>
#include <Poco/Net/PartHandler.h>
#include <Core/Block.h>
#include <Client/Connection.h>
#include <IO/ReadBuffer.h>
namespace Poco
{
namespace Net
{
class NameValueCollection;
class MessageHeader;
}
}
namespace boost
{
namespace program_options
{
class variables_map;
}
}
namespace DB
{
class Context;
/// The base class containing the basic information about external table and
/// basic functions for extracting this information from text fields.
class BaseExternalTable
{
public:
std::string file; /// File with data or '-' if stdin
std::string name; /// The name of the table
std::string format; /// Name of the data storage format
/// Description of the table structure: (column name, data type name)
std::vector<std::pair<std::string, std::string>> structure;
std::unique_ptr<ReadBuffer> read_buffer;
Block sample_block;
virtual ~BaseExternalTable() {}
/// Initialize read_buffer, depending on the data source. By default, does nothing.
virtual void initReadBuffer() {}
/// Get the table data - a pair (a stream with the contents of the table, the name of the table)
ExternalTableData getData(const Context & context);
protected:
/// Clear all accumulated information
void clean();
/// Function for debugging information output
void write();
static std::vector<std::string> split(const std::string & s, const std::string & d);
/// Construct the `structure` vector from the text field `structure`
virtual void parseStructureFromStructureField(const std::string & argument);
/// Construct the `structure` vector from the text field `types`
virtual void parseStructureFromTypesField(const std::string & argument);
private:
/// Initialize sample_block according to the structure of the table stored in the `structure`
void initSampleBlock();
};
/// Parsing of external table used in the tcp client.
class ExternalTable : public BaseExternalTable
{
public:
void initReadBuffer() override;
/// Extract parameters from variables_map, which is built on the client command line
ExternalTable(const boost::program_options::variables_map & external_options);
};
/// Parsing of external table used when sending tables via http
/// The `handlePart` function will be called for each table passed,
/// so it's also necessary to call `clean` at the end of the `handlePart`.
class ExternalTablesHandler : public Poco::Net::PartHandler, BaseExternalTable
{
public:
ExternalTablesHandler(Context & context_, const Poco::Net::NameValueCollection & params_) : context(context_), params(params_) {}
void handlePart(const Poco::Net::MessageHeader & header, std::istream & stream);
private:
Context & context;
const Poco::Net::NameValueCollection & params;
std::unique_ptr<ReadBuffer> read_buffer_impl;
};
}

6
dbms/src/Core/Protocol.h
View File

@ -69,6 +69,7 @@ namespace Protocol
Totals = 7, /// A block with totals (compressed or not).
Extremes = 8, /// A block with minimums and maximums (compressed or not).
TablesStatusResponse = 9, /// A response to TablesStatus request.
Log = 10 /// System logs of the query execution
};
/// NOTE: If the type of packet argument would be Enum, the comparison packet >= 0 && packet < 10
@ -77,8 +78,8 @@ namespace Protocol
/// See https://www.securecoding.cert.org/confluence/display/cplusplus/INT36-CPP.+Do+not+use+out-of-range+enumeration+values
inline const char * toString(UInt64 packet)
{
static const char * data[] = { "Hello", "Data", "Exception", "Progress", "Pong", "EndOfStream", "ProfileInfo", "Totals", "Extremes", "TablesStatusResponse" };
return packet < 10
static const char * data[] = { "Hello", "Data", "Exception", "Progress", "Pong", "EndOfStream", "ProfileInfo", "Totals", "Extremes", "TablesStatusResponse", "Log" };
return packet < 11
? data[packet]
: "Unknown packet";
}
@ -97,6 +98,7 @@ namespace Protocol
Cancel = 3, /// Cancel the query execution.
Ping = 4, /// Check that connection to the server is alive.
TablesStatusRequest = 5, /// Check status of tables on the server.
KeepAlive = 6 /// Keep the connection alive
};
inline const char * toString(UInt64 packet)

4
dbms/src/Core/Types.h
View File

@ -120,9 +120,8 @@ namespace DB
{
/// Own FieldType for Decimal
template <typename T>
class Dec
struct Dec
{
public:
using NativeType = T;
Dec() = default;
@ -149,7 +148,6 @@ namespace DB
const Dec<T> & operator /= (const T & x) { value /= x; return *this; }
const Dec<T> & operator %= (const T & x) { value %= x; return *this; }
private:
T value;
};

84
dbms/src/DataStreams/AsynchronousBlockInputStream.cpp Normal file
View File

@ -0,0 +1,84 @@
#include "AsynchronousBlockInputStream.h"
#include <Common/CurrentThread.h>
namespace DB
{
Block AsynchronousBlockInputStream::readImpl()
{
/// If there were no calculations yet, calculate the first block synchronously
if (!started)
{
calculate();
started = true;
}
else /// If the calculations are already in progress - wait for the result
pool.wait();
if (exception)
std::rethrow_exception(exception);
Block res = block;
if (!res)
return res;
/// Start the next block calculation
block.clear();
next();
return res;
}
void AsynchronousBlockInputStream::next()
{
ready.reset();
pool.schedule([this, thread_group=CurrentThread::getGroup()] ()
{
CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread};
try
{
if (first)
setThreadName("AsyncBlockInput");
/// AsynchronousBlockInputStream is used in Client which does not create queries and thread groups
if (thread_group)
CurrentThread::attachToIfDetached(thread_group);
}
catch (...)
{
exception = std::current_exception();
ready.set();
return;
}
calculate();
});
}
void AsynchronousBlockInputStream::calculate()
{
try
{
if (first)
{
first = false;
children.back()->readPrefix();
}
block = children.back()->read();
}
catch (...)
{
exception = std::current_exception();
}
ready.set();
}
}

59
dbms/src/DataStreams/AsynchronousBlockInputStream.h
View File

@ -7,6 +7,7 @@
#include <Common/CurrentMetrics.h>
#include <common/ThreadPool.h>
#include <Common/MemoryTracker.h>
#include <Poco/Ext/ThreadNumber.h>
namespace CurrentMetrics
@ -91,64 +92,12 @@ protected:
Block block;
std::exception_ptr exception;
Block readImpl() override;
Block readImpl() override
{
/// If there were no calculations yet, calculate the first block synchronously
if (!started)
{
calculate(current_memory_tracker);
started = true;
}
else /// If the calculations are already in progress - wait for the result
pool.wait();
if (exception)
std::rethrow_exception(exception);
Block res = block;
if (!res)
return res;
/// Start the next block calculation
block.clear();
next();
return res;
}
void next()
{
ready.reset();
pool.schedule(std::bind(&AsynchronousBlockInputStream::calculate, this, current_memory_tracker));
}
void next();
/// Calculations that can be performed in a separate thread
void calculate(MemoryTracker * memory_tracker)
{
CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread};
try
{
if (first)
{
first = false;
setThreadName("AsyncBlockInput");
current_memory_tracker = memory_tracker;
children.back()->readPrefix();
}
block = children.back()->read();
}
catch (...)
{
exception = std::current_exception();
}
ready.set();
}
void calculate();
};
}

16
dbms/src/DataStreams/BlockIO.h
View File

@ -12,8 +12,7 @@ class ProcessListEntry;
struct BlockIO
{
/** process_list_entry should be destroyed after in and after out,
* since in and out contain pointer to an object inside process_list_entry
* (MemoryTracker * current_memory_tracker),
* since in and out contain pointer to objects inside process_list_entry (query-level MemoryTracker for example),
* which could be used before destroying of in and out.
*/
std::shared_ptr<ProcessListEntry> process_list_entry;
@ -38,12 +37,17 @@ struct BlockIO
exception_callback();
}
/// We provide the correct order of destruction.
void reset()
{
out.reset();
in.reset();
process_list_entry.reset();
}
BlockIO & operator= (const BlockIO & rhs)
{
/// We provide the correct order of destruction.
out = nullptr;
in = nullptr;
process_list_entry = nullptr;
reset();
process_list_entry = rhs.process_list_entry;
in = rhs.in;

4
dbms/src/DataStreams/CountingBlockOutputStream.h
View File

@ -20,7 +20,7 @@ public:
progress_callback = callback;
}
void setProcessListElement(ProcessListElement * elem)
void setProcessListElement(QueryStatus * elem)
{
process_elem = elem;
}
@ -43,7 +43,7 @@ protected:
BlockOutputStreamPtr stream;
Progress progress;
ProgressCallback progress_callback;
ProcessListElement * process_elem = nullptr;
QueryStatus * process_elem = nullptr;
};
}

27
dbms/src/DataStreams/FilterBlockInputStream.cpp
View File

@ -17,8 +17,9 @@ namespace ErrorCodes
}
FilterBlockInputStream::FilterBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_, const String & filter_column_name)
: expression(expression_)
FilterBlockInputStream::FilterBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_,
const String & filter_column_name, bool remove_filter)
: remove_filter(remove_filter), expression(expression_)
{
children.push_back(input);
@ -40,6 +41,9 @@ FilterBlockInputStream::FilterBlockInputStream(const BlockInputStreamPtr & input
FilterDescription filter_description_check(*column_elem.column);
column_elem.column = column_elem.type->createColumnConst(header.rows(), UInt64(1));
}
if (remove_filter)
header.erase(filter_column_name);
}
@ -69,7 +73,7 @@ Block FilterBlockInputStream::readImpl()
Block res;
if (constant_filter_description.always_false)
return res;
return removeFilterIfNeed(std::move(res));
/// Until non-empty block after filtering or end of stream.
while (1)
@ -81,7 +85,7 @@ Block FilterBlockInputStream::readImpl()
expression->execute(res);
if (constant_filter_description.always_true)
return res;
return removeFilterIfNeed(std::move(res));
size_t columns = res.columns();
ColumnPtr column = res.safeGetByPosition(filter_column).column;
@ -100,7 +104,7 @@ Block FilterBlockInputStream::readImpl()
}
if (constant_filter_description.always_true)
return res;
return removeFilterIfNeed(std::move(res));
FilterDescription filter_and_holder(*column);
@ -142,7 +146,7 @@ Block FilterBlockInputStream::readImpl()
/// Replace the column with the filter by a constant.
res.safeGetByPosition(filter_column).column = res.safeGetByPosition(filter_column).type->createColumnConst(filtered_rows, UInt64(1));
/// No need to touch the rest of the columns.
return res;
return removeFilterIfNeed(std::move(res));
}
/// Filter the rest of the columns.
@ -170,9 +174,18 @@ Block FilterBlockInputStream::readImpl()
current_column.column = current_column.column->filter(*filter_and_holder.data, -1);
}
return res;
return removeFilterIfNeed(std::move(res));
}
}
Block FilterBlockInputStream::removeFilterIfNeed(Block && block)
{
if (block && remove_filter)
block.erase(static_cast<size_t>(filter_column));
return std::move(block);
}
}

7
dbms/src/DataStreams/FilterBlockInputStream.h
View File

@ -20,7 +20,8 @@ private:
using ExpressionActionsPtr = std::shared_ptr<ExpressionActions>;
public:
FilterBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_, const String & filter_column_name_);
FilterBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_,
const String & filter_column_name_, bool remove_filter = false);
String getName() const override;
Block getTotals() override;
@ -29,12 +30,16 @@ public:
protected:
Block readImpl() override;
bool remove_filter;
private:
ExpressionActionsPtr expression;
Block header;
ssize_t filter_column;
ConstantFilterDescription constant_filter_description;
Block removeFilterIfNeed(Block && block);
};
}

36
dbms/src/DataStreams/IProfilingBlockInputStream.cpp
View File

@ -1,6 +1,13 @@
#include <Interpreters/Quota.h>
#include <Interpreters/ProcessList.h>
#include <DataStreams/IProfilingBlockInputStream.h>
#include <Common/CurrentThread.h>
namespace ProfileEvents
{
extern const Event ThrottlerSleepMicroseconds;
}
namespace DB
@ -286,21 +293,34 @@ void IProfilingBlockInputStream::progressImpl(const Progress & value)
size_t total_rows = progress.total_rows;
if (limits.min_execution_speed || (total_rows && limits.timeout_before_checking_execution_speed != 0))
{
double total_elapsed = info.total_stopwatch.elapsedSeconds();
constexpr UInt64 profile_events_update_period_microseconds = 10 * 1000; // 10 milliseconds
UInt64 total_elapsed_microseconds = info.total_stopwatch.elapsedMicroseconds();
if (total_elapsed > limits.timeout_before_checking_execution_speed.totalMicroseconds() / 1000000.0)
if (last_profile_events_update_time + profile_events_update_period_microseconds < total_elapsed_microseconds)
{
CurrentThread::updatePerformanceCounters();
last_profile_events_update_time = total_elapsed_microseconds;
}
if ((limits.min_execution_speed || (total_rows && limits.timeout_before_checking_execution_speed != 0))
&& (static_cast<Int64>(total_elapsed_microseconds) > limits.timeout_before_checking_execution_speed.totalMicroseconds()))
{
/// Do not count sleeps in throttlers
UInt64 throttler_sleep_microseconds = CurrentThread::getProfileEvents()[ProfileEvents::ThrottlerSleepMicroseconds];
double elapsed_seconds = (throttler_sleep_microseconds > total_elapsed_microseconds)
? 0.0 : (total_elapsed_microseconds - throttler_sleep_microseconds) / 1000000.0;
if (elapsed_seconds > 0)
{
if (limits.min_execution_speed && progress.rows / total_elapsed < limits.min_execution_speed)
throw Exception("Query is executing too slow: " + toString(progress.rows / total_elapsed)
if (limits.min_execution_speed && progress.rows / elapsed_seconds < limits.min_execution_speed)
throw Exception("Query is executing too slow: " + toString(progress.rows / elapsed_seconds)
+ " rows/sec., minimum: " + toString(limits.min_execution_speed),
ErrorCodes::TOO_SLOW);
/// If the predicted execution time is longer than `max_execution_time`.
if (limits.max_execution_time != 0 && total_rows)
{
double estimated_execution_time_seconds = total_elapsed * (static_cast<double>(total_rows) / progress.rows);
double estimated_execution_time_seconds = elapsed_seconds * (static_cast<double>(total_rows) / progress.rows);
if (estimated_execution_time_seconds > limits.max_execution_time.totalSeconds())
throw Exception("Estimated query execution time (" + toString(estimated_execution_time_seconds) + " seconds)"
@ -363,7 +383,7 @@ void IProfilingBlockInputStream::setProgressCallback(const ProgressCallback & ca
}
void IProfilingBlockInputStream::setProcessListElement(ProcessListElement * elem)
void IProfilingBlockInputStream::setProcessListElement(QueryStatus * elem)
{
process_list_elem = elem;

7
dbms/src/DataStreams/IProfilingBlockInputStream.h
View File

@ -20,6 +20,7 @@ namespace ErrorCodes
}
class QuotaForIntervals;
class QueryStatus;
class ProcessListElement;
class IProfilingBlockInputStream;
@ -103,7 +104,7 @@ public:
* Based on this information, the quota and some restrictions will be checked.
* This information will also be available in the SHOW PROCESSLIST request.
*/
void setProcessListElement(ProcessListElement * elem);
void setProcessListElement(QueryStatus * elem);
/** Set the approximate total number of rows to read.
*/
@ -178,7 +179,9 @@ protected:
std::atomic<bool> is_cancelled{false};
std::atomic<bool> is_killed{false};
ProgressCallback progress_callback;
ProcessListElement * process_list_elem = nullptr;
QueryStatus * process_list_elem = nullptr;
/// According to total_stopwatch in microseconds
UInt64 last_profile_events_update_time = 0;
/// Additional information that can be generated during the work process.

82
dbms/src/DataStreams/InternalTextLogsRowOutputStream.cpp Normal file
View File

@ -0,0 +1,82 @@
#include "InternalTextLogsRowOutputStream.h"
#include <Core/Block.h>
#include <Interpreters/InternalTextLogsQueue.h>
#include <Common/typeid_cast.h>
#include <DataTypes/IDataType.h>
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnString.h>
#include <IO/WriteHelpers.h>
namespace DB
{
Block InternalTextLogsRowOutputStream::getHeader() const
{
return InternalTextLogsQueue::getSampleBlock();
}
void InternalTextLogsRowOutputStream::write(const Block & block)
{
auto & array_event_time = typeid_cast<const ColumnUInt32 &>(*block.getByName("event_time").column).getData();
auto & array_microseconds = typeid_cast<const ColumnUInt32 &>(*block.getByName("event_time_microseconds").column).getData();
auto & column_host_name = typeid_cast<const ColumnString &>(*block.getByName("host_name").column);
auto & column_query_id = typeid_cast<const ColumnString &>(*block.getByName("query_id").column);
auto & array_thread_number = typeid_cast<const ColumnUInt32 &>(*block.getByName("thread_number").column).getData();
auto & array_priority = typeid_cast<const ColumnInt8 &>(*block.getByName("priority").column).getData();
auto & column_source = typeid_cast<const ColumnString &>(*block.getByName("source").column);
auto & column_text = typeid_cast<const ColumnString &>(*block.getByName("text").column);
for (size_t row_num = 0; row_num < block.rows(); ++row_num)
{
auto host_name = column_host_name.getDataAt(row_num);
if (host_name.size)
{
writeCString("[", wb);
writeString(host_name, wb);
writeCString("] ", wb);
}
auto event_time = array_event_time[row_num];
writeDateTimeText<'.', ':'>(event_time, wb);
auto microseconds = array_microseconds[row_num];
writeChar('.', wb);
writeChar('0' + ((microseconds / 100000) % 10), wb);
writeChar('0' + ((microseconds / 10000) % 10), wb);
writeChar('0' + ((microseconds / 1000) % 10), wb);
writeChar('0' + ((microseconds / 100) % 10), wb);
writeChar('0' + ((microseconds / 10) % 10), wb);
writeChar('0' + ((microseconds / 1) % 10), wb);
auto query_id = column_query_id.getDataAt(row_num);
if (query_id.size)
{
writeCString(" {", wb);
writeString(query_id, wb);
writeCString("}", wb);
}
UInt32 thread_number = array_thread_number[row_num];
writeCString(" [ ", wb);
writeIntText(thread_number, wb);
writeCString(" ] <", wb);
Int8 priority = array_priority[row_num];
writeString(InternalTextLogsQueue::getPriorityName(priority), wb);
writeCString("> ", wb);
auto source = column_source.getDataAt(row_num);
writeString(source, wb);
writeCString(": ", wb);
auto text = column_text.getDataAt(row_num);
writeString(text, wb);
writeChar('\n', wb);
}
}
}

32
dbms/src/DataStreams/InternalTextLogsRowOutputStream.h Normal file
View File

@ -0,0 +1,32 @@
#pragma once
#include <DataStreams/IBlockOutputStream.h>
#include <IO/WriteBuffer.h>
namespace DB
{
/// Prints internal server logs
/// Input blocks have to have the same structure as SystemLogsQueue::getSampleBlock()
/// NOTE: IRowOutputStream does not suite well for this case
class InternalTextLogsRowOutputStream : public IBlockOutputStream
{
public:
InternalTextLogsRowOutputStream(WriteBuffer & buf_out) : wb(buf_out) {}
Block getHeader() const override;
void write(const Block & block) override;
void flush() override
{
wb.next();
}
private:
WriteBuffer & wb;
};
}

24
dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp
View File

@ -3,6 +3,7 @@
#include <Common/CurrentMetrics.h>
#include <Common/MemoryTracker.h>
#include <DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h>
#include <Common/CurrentThread.h>
namespace CurrentMetrics
@ -175,10 +176,10 @@ void MergingAggregatedMemoryEfficientBlockInputStream::start()
{
auto & child = children[i];
auto memory_tracker = current_memory_tracker;
reading_pool->schedule([&child, memory_tracker]
auto thread_group = CurrentThread::getGroup();
reading_pool->schedule([&child, thread_group]
{
current_memory_tracker = memory_tracker;
CurrentThread::attachToIfDetached(thread_group);
setThreadName("MergeAggReadThr");
CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread};
child->readPrefix();
@ -196,8 +197,7 @@ void MergingAggregatedMemoryEfficientBlockInputStream::start()
*/
for (size_t i = 0; i < merging_threads; ++i)
pool.schedule(std::bind(&MergingAggregatedMemoryEfficientBlockInputStream::mergeThread,
this, current_memory_tracker));
pool.schedule([this, thread_group=CurrentThread::getGroup()] () { mergeThread(thread_group); } );
}
}
@ -293,14 +293,16 @@ void MergingAggregatedMemoryEfficientBlockInputStream::finalize()
}
void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(MemoryTracker * memory_tracker)
void MergingAggregatedMemoryEfficientBlockInputStream::mergeThread(ThreadGroupStatusPtr thread_group)
{
setThreadName("MergeAggMergThr");
current_memory_tracker = memory_tracker;
CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread};
try
{
if (thread_group)
CurrentThread::attachTo(thread_group);
setThreadName("MergeAggMergThr");
while (!parallel_merge_data->finish)
{
/** Receiving next blocks is processing by one thread pool, and merge is in another.
@ -480,10 +482,10 @@ MergingAggregatedMemoryEfficientBlockInputStream::BlocksToMerge MergingAggregate
{
if (need_that_input(input))
{
auto memory_tracker = current_memory_tracker;
reading_pool->schedule([&input, &read_from_input, memory_tracker]
auto thread_group = CurrentThread::getGroup();
reading_pool->schedule([&input, &read_from_input, thread_group]
{
current_memory_tracker = memory_tracker;
CurrentThread::attachToIfDetached(thread_group);
setThreadName("MergeAggReadThr");
CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread};
read_from_input(input);

3
dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h
View File

@ -3,6 +3,7 @@
#include <Interpreters/Aggregator.h>
#include <DataStreams/IProfilingBlockInputStream.h>
#include <Common/ConcurrentBoundedQueue.h>
#include <Common/CurrentThread.h>
#include <common/ThreadPool.h>
#include <condition_variable>
@ -151,7 +152,7 @@ private:
std::unique_ptr<ParallelMergeData> parallel_merge_data;
void mergeThread(MemoryTracker * memory_tracker);
void mergeThread(ThreadGroupStatusPtr main_thread);
void finalize();
};

12
dbms/src/DataStreams/ParallelInputsProcessor.h
View File

@ -12,6 +12,7 @@
#include <Common/setThreadName.h>
#include <Common/CurrentMetrics.h>
#include <Common/MemoryTracker.h>
#include <Common/CurrentThread.h>
/** Allows to process multiple block input streams (sources) in parallel, using specified number of threads.
@ -105,8 +106,9 @@ public:
{
active_threads = max_threads;
threads.reserve(max_threads);
auto thread_group = CurrentThread::getGroup();
for (size_t i = 0; i < max_threads; ++i)
threads.emplace_back(std::bind(&ParallelInputsProcessor::thread, this, current_memory_tracker, i));
threads.emplace_back([=] () { thread(thread_group, i); } );
}
/// Ask all sources to stop earlier than they run out.
@ -174,16 +176,16 @@ private:
}
}
void thread(MemoryTracker * memory_tracker, size_t thread_num)
void thread(ThreadGroupStatusPtr thread_group, size_t thread_num)
{
current_memory_tracker = memory_tracker;
std::exception_ptr exception;
setThreadName("ParalInputsProc");
CurrentMetrics::Increment metric_increment{CurrentMetrics::QueryThread};
try
{
setThreadName("ParalInputsProc");
CurrentThread::attachTo(thread_group);
while (!finish)
{
InputData unprepared_input;

10
dbms/src/DataStreams/RemoteBlockInputStream.cpp
View File

@ -1,8 +1,10 @@
#include <DataStreams/RemoteBlockInputStream.h>
#include <DataStreams/OneBlockInputStream.h>
#include <Common/NetException.h>
#include <Common/CurrentThread.h>
#include <Interpreters/Context.h>
#include <Interpreters/castColumn.h>
#include <Interpreters/InternalTextLogsQueue.h>
#include <Storages/IStorage.h>
@ -137,7 +139,7 @@ void RemoteBlockInputStream::sendExternalTables()
for (const auto & table : external_tables)
{
StoragePtr cur = table.second;
QueryProcessingStage::Enum read_from_table_stage = QueryProcessingStage::Complete;
QueryProcessingStage::Enum read_from_table_stage = cur->getQueryProcessingStage(context);
BlockInputStreams input = cur->read(cur->getColumns().getNamesOfPhysical(), {}, context,
read_from_table_stage, DEFAULT_BLOCK_SIZE, 1);
if (input.size() == 0)
@ -232,6 +234,12 @@ Block RemoteBlockInputStream::readImpl()
extremes = packet.block;
break;
case Protocol::Server::Log:
/// Pass logs from remote server to client
if (auto log_queue = CurrentThread::getInternalTextLogsQueue())
log_queue->pushBlock(std::move(packet.block));
break;
default:
got_unknown_packet_from_replica = true;
throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER);

75
dbms/src/DataStreams/RemoteBlockOutputStream.cpp
View File

@ -4,6 +4,8 @@
#include <common/logger_useful.h>
#include <Common/NetException.h>
#include <Common/CurrentThread.h>
#include <Interpreters/InternalTextLogsQueue.h>
namespace DB
@ -24,23 +26,33 @@ RemoteBlockOutputStream::RemoteBlockOutputStream(Connection & connection_, const
*/
connection.sendQuery(query, "", QueryProcessingStage::Complete, settings, nullptr);
Connection::Packet packet = connection.receivePacket();
if (Protocol::Server::Data == packet.type)
while (true)
{
header = packet.block;
Connection::Packet packet = connection.receivePacket();
if (!header)
throw Exception("Logical error: empty block received as table structure", ErrorCodes::LOGICAL_ERROR);
if (Protocol::Server::Data == packet.type)
{
header = packet.block;
if (!header)
throw Exception("Logical error: empty block received as table structure", ErrorCodes::LOGICAL_ERROR);
break;
}
else if (Protocol::Server::Exception == packet.type)
{
packet.exception->rethrow();
break;
}
else if (Protocol::Server::Log == packet.type)
{
/// Pass logs from remote server to client
if (auto log_queue = CurrentThread::getInternalTextLogsQueue())
log_queue->pushBlock(std::move(packet.block));
}
else
throw NetException("Unexpected packet from server (expected Data or Exception, got "
+ String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER);
}
else if (Protocol::Server::Exception == packet.type)
{
packet.exception->rethrow();
return;
}
else
throw NetException("Unexpected packet from server (expected Data or Exception, got "
+ String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER);
}
@ -55,15 +67,11 @@ void RemoteBlockOutputStream::write(const Block & block)
catch (const NetException &)
{
/// Try to get more detailed exception from server
if (connection.poll(0))
auto packet_type = connection.checkPacket();
if (packet_type && *packet_type == Protocol::Server::Exception)
{
Connection::Packet packet = connection.receivePacket();
if (Protocol::Server::Exception == packet.type)
{
packet.exception->rethrow();
return;
}
packet.exception->rethrow();
}
throw;
@ -83,18 +91,23 @@ void RemoteBlockOutputStream::writeSuffix()
/// Empty block means end of data.
connection.sendData(Block());
/// Receive EndOfStream packet.
Connection::Packet packet = connection.receivePacket();
if (Protocol::Server::EndOfStream == packet.type)
/// Wait for EndOfStream or Exception packet, skip Log packets.
while (true)
{
/// Do nothing.
Connection::Packet packet = connection.receivePacket();
if (Protocol::Server::EndOfStream == packet.type)
break;
else if (Protocol::Server::Exception == packet.type)
packet.exception->rethrow();
else if (Protocol::Server::Log == packet.type)
{
// Do nothing
}
else
throw NetException("Unexpected packet from server (expected EndOfStream or Exception, got "
+ String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER);
}
else if (Protocol::Server::Exception == packet.type)
packet.exception->rethrow();
else
throw NetException("Unexpected packet from server (expected EndOfStream or Exception, got "
+ String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER);
finished = true;
}

2
dbms/src/DataStreams/tests/expression_stream.cpp
View File

@ -47,7 +47,7 @@ try
Names column_names;
column_names.push_back("number");
QueryProcessingStage::Enum stage;
QueryProcessingStage::Enum stage = table->getQueryProcessingStage(context);
BlockInputStreamPtr in;
in = table->read(column_names, {}, context, stage, 8192, 1)[0];

2
dbms/src/DataStreams/tests/filter_stream.cpp
View File

@ -52,7 +52,7 @@ try
Names column_names;
column_names.push_back("number");
QueryProcessingStage::Enum stage;
QueryProcessingStage::Enum stage = table->getQueryProcessingStage(context);
BlockInputStreamPtr in = table->read(column_names, {}, context, stage, 8192, 1)[0];
in = std::make_shared<FilterBlockInputStream>(in, expression, "equals(modulo(number, 3), 1)");

0
dbms/src/DataStreams/tests/filter_stream_hitlog.cpp Normal file
View File

0
dbms/src/DataStreams/tests/native_streams.cpp Normal file
View File

0
dbms/src/DataStreams/tests/sorting_stream.cpp Normal file
View File

2
dbms/src/DataStreams/tests/union_stream2.cpp
View File

@ -34,7 +34,7 @@ try
StoragePtr table = context.getTable("default", "hits6");
QueryProcessingStage::Enum stage;
QueryProcessingStage::Enum stage = table->getQueryProcessingStage(context);
BlockInputStreams streams = table->read(column_names, {}, context, stage, settings.max_block_size, settings.max_threads);
for (size_t i = 0, size = streams.size(); i < size; ++i)

Some files were not shown because too many files have changed in this diff Show More