From 6ff9dfce34f73fa993b2e49f0f272155f490cbbd Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 8 Jul 2019 14:41:54 +0300 Subject: [PATCH 01/64] first try --- dbms/programs/server/TCPHandler.cpp | 4 +++- dbms/src/Core/Settings.h | 2 +- .../0_stateless/00965_logs_level_bugfix.reference | 9 +++++++++ .../queries/0_stateless/00965_logs_level_bugfix.sh | 11 +++++++++++ libs/libcommon/include/common/logger_useful.h | 14 +++++++++----- 5 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00965_logs_level_bugfix.reference create mode 100755 dbms/tests/queries/0_stateless/00965_logs_level_bugfix.sh diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index bfbc0a590a2..9543051796f 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -28,6 +28,7 @@ #include #include #include +#include <../../../libs/libcommon/include/common/logger_useful.h> #include "TCPHandler.h" @@ -170,8 +171,9 @@ void TCPHandler::runImpl() send_exception_with_stack_trace = query_context->getSettingsRef().calculate_text_stack_trace; /// Should we send internal logs to client? + CLIENT_LOGS_LEVEL = query_context->getSettingsRef().send_logs_level.value; if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_LOGS - && query_context->getSettingsRef().send_logs_level.value != LogsLevel::none) + && CLIENT_LOGS_LEVEL != LogsLevel::none) { state.logs_queue = std::make_shared(); state.logs_queue->max_priority = Poco::Logger::parseLevel(query_context->getSettingsRef().send_logs_level.toString()); diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index fabd4693e33..541481d3d8b 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -288,7 +288,7 @@ struct Settings : public SettingsCollection M(SettingOverflowMode, distinct_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \ \ M(SettingUInt64, max_memory_usage, 0, "Maximum memory usage for processing of single query. Zero means unlimited.") \ - M(SettingUInt64, max_memory_usage_for_user, 0, "Maximum memory usage for processing all concurrently running queries for the user. Zero means unlimited.") \ + M(SettingUInt64, max_memory_usage_for_user, 0, "Maximum memory usage for procndessing all concurrently running queries for the user. Zero means unlimited.") \ M(SettingUInt64, max_memory_usage_for_all_queries, 0, "Maximum memory usage for processing all concurrently running queries on the server. Zero means unlimited.") \ \ M(SettingUInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.") \ diff --git a/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.reference b/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.reference new file mode 100644 index 00000000000..eee8323a649 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.reference @@ -0,0 +1,9 @@ +1 + + + + + + + + diff --git a/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.sh b/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.sh new file mode 100755 index 00000000000..8037a9c3d8e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +cp /dev/null 00965_logs_level_bugfix.tmp + +clickhouse-client --send_logs_level="debug" --query="SELECT 1;" 2>> 00965_logs_level_bugfix.tmp +awk '{ print $8 }' 00965_logs_level_bugfix.tmp + + diff --git a/libs/libcommon/include/common/logger_useful.h b/libs/libcommon/include/common/logger_useful.h index 245a79c7982..638621d3572 100644 --- a/libs/libcommon/include/common/logger_useful.h +++ b/libs/libcommon/include/common/logger_useful.h @@ -4,41 +4,45 @@ #include #include +#include <../../../../dbms/src/Core/SettingsCommon.h> #ifndef QUERY_PREVIEW_LENGTH #define QUERY_PREVIEW_LENGTH 160 #endif using Poco::Logger; +using DB::LogsLevel; + +POCO_UNUSED static LogsLevel CLIENT_LOGS_LEVEL = LogsLevel::none; /// Logs a message to a specified logger with that level. #define LOG_TRACE(logger, message) do { \ - if ((logger)->trace()) {\ + if ((logger)->trace() || CLIENT_LOGS_LEVEL >= LogsLevel::trace) {\ std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ (logger)->trace(oss_internal_rare.str());}} while(false) #define LOG_DEBUG(logger, message) do { \ - if ((logger)->debug()) {\ + if ((logger)->debug() || CLIENT_LOGS_LEVEL >= LogsLevel::debug) {\ std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ (logger)->debug(oss_internal_rare.str());}} while(false) #define LOG_INFO(logger, message) do { \ - if ((logger)->information()) {\ + if ((logger)->information() || CLIENT_LOGS_LEVEL >= LogsLevel::information) {\ std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ (logger)->information(oss_internal_rare.str());}} while(false) #define LOG_WARNING(logger, message) do { \ - if ((logger)->warning()) {\ + if ((logger)->warning() || CLIENT_LOGS_LEVEL >= LogsLevel::warning) {\ std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ (logger)->warning(oss_internal_rare.str());}} while(false) #define LOG_ERROR(logger, message) do { \ - if ((logger)->error()) {\ + if ((logger)->error() || CLIENT_LOGS_LEVEL >= LogsLevel::error) {\ std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ (logger)->error(oss_internal_rare.str());}} while(false) From 1740a24ded712677a613de0a7b1c29d17a7980d8 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 8 Jul 2019 14:44:07 +0300 Subject: [PATCH 02/64] better --- dbms/src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index 541481d3d8b..fabd4693e33 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -288,7 +288,7 @@ struct Settings : public SettingsCollection M(SettingOverflowMode, distinct_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.") \ \ M(SettingUInt64, max_memory_usage, 0, "Maximum memory usage for processing of single query. Zero means unlimited.") \ - M(SettingUInt64, max_memory_usage_for_user, 0, "Maximum memory usage for procndessing all concurrently running queries for the user. Zero means unlimited.") \ + M(SettingUInt64, max_memory_usage_for_user, 0, "Maximum memory usage for processing all concurrently running queries for the user. Zero means unlimited.") \ M(SettingUInt64, max_memory_usage_for_all_queries, 0, "Maximum memory usage for processing all concurrently running queries on the server. Zero means unlimited.") \ \ M(SettingUInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.") \ From c0a4495d1799095c49ad8d39ba8568aa6fb9f2d2 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 8 Jul 2019 17:15:48 +0300 Subject: [PATCH 03/64] multithreading --- .../0_stateless/00965_logs_level_bugfix.reference | 15 +++++++++++++++ .../0_stateless/00965_logs_level_bugfix.sh | 8 +++++++- libs/libcommon/include/common/logger_useful.h | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.reference b/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.reference index eee8323a649..233d012cabe 100644 --- a/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.reference +++ b/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.reference @@ -1,4 +1,18 @@ 1 +1 +1 +1 +1 +1 + + + + + + + + + @@ -7,3 +21,4 @@ + diff --git a/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.sh b/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.sh index 8037a9c3d8e..50d1df6ce80 100755 --- a/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.sh +++ b/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.sh @@ -5,7 +5,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) cp /dev/null 00965_logs_level_bugfix.tmp +clickhouse-client --send_logs_level="trace" --query="SELECT 1;" 2>> 00965_logs_level_bugfix.tmp clickhouse-client --send_logs_level="debug" --query="SELECT 1;" 2>> 00965_logs_level_bugfix.tmp +clickhouse-client --send_logs_level="information" --query="SELECT 1;" 2>> 00965_logs_level_bugfix.tmp +clickhouse-client --send_logs_level="warning" --query="SELECT 1;" 2>> 00965_logs_level_bugfix.tmp +clickhouse-client --send_logs_level="error" --query="SELECT 1;" 2>> 00965_logs_level_bugfix.tmp +clickhouse-client --send_logs_level="none" --query="SELECT 1;" 2>> 00965_logs_level_bugfix.tmp + awk '{ print $8 }' 00965_logs_level_bugfix.tmp - + diff --git a/libs/libcommon/include/common/logger_useful.h b/libs/libcommon/include/common/logger_useful.h index 638621d3572..fc99c6ad864 100644 --- a/libs/libcommon/include/common/logger_useful.h +++ b/libs/libcommon/include/common/logger_useful.h @@ -13,7 +13,7 @@ using Poco::Logger; using DB::LogsLevel; -POCO_UNUSED static LogsLevel CLIENT_LOGS_LEVEL = LogsLevel::none; +POCO_UNUSED std::atomic CLIENT_LOGS_LEVEL = LogsLevel::none; /// Logs a message to a specified logger with that level. From f3e026edf58e4aa0489f1d48a3db30dfec8fefc6 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 8 Jul 2019 17:49:04 +0300 Subject: [PATCH 04/64] static missed --- libs/libcommon/include/common/logger_useful.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/libcommon/include/common/logger_useful.h b/libs/libcommon/include/common/logger_useful.h index fc99c6ad864..6695f851e98 100644 --- a/libs/libcommon/include/common/logger_useful.h +++ b/libs/libcommon/include/common/logger_useful.h @@ -13,7 +13,7 @@ using Poco::Logger; using DB::LogsLevel; -POCO_UNUSED std::atomic CLIENT_LOGS_LEVEL = LogsLevel::none; +POCO_UNUSED static std::atomic CLIENT_LOGS_LEVEL = LogsLevel::none; /// Logs a message to a specified logger with that level. From 55e9e21cc5bdb24e91344397c1743967ff5c6dd4 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 8 Jul 2019 19:11:38 +0300 Subject: [PATCH 05/64] better includes --- dbms/programs/server/TCPHandler.cpp | 2 +- libs/libcommon/include/common/logger_useful.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 9543051796f..990cb582828 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -28,7 +28,7 @@ #include #include #include -#include <../../../libs/libcommon/include/common/logger_useful.h> +#include #include "TCPHandler.h" diff --git a/libs/libcommon/include/common/logger_useful.h b/libs/libcommon/include/common/logger_useful.h index 6695f851e98..a065646a4f8 100644 --- a/libs/libcommon/include/common/logger_useful.h +++ b/libs/libcommon/include/common/logger_useful.h @@ -3,8 +3,9 @@ /// Macros for convenient usage of Poco logger. #include +#include #include -#include <../../../../dbms/src/Core/SettingsCommon.h> +#include #ifndef QUERY_PREVIEW_LENGTH #define QUERY_PREVIEW_LENGTH 160 From 21ad247df304593af4337d2ee64ccbd5916d81c5 Mon Sep 17 00:00:00 2001 From: NIKITA MIKHAILOV Date: Tue, 9 Jul 2019 13:39:05 +0300 Subject: [PATCH 06/64] only for saving this changes --- dbms/programs/server/TCPHandler.cpp | 6 +++--- dbms/src/Common/CurrentThread.cpp | 5 +++-- dbms/src/Common/CurrentThread.h | 3 ++- dbms/src/Common/ThreadStatus.cpp | 4 +++- dbms/src/Common/ThreadStatus.h | 6 +++++- libs/libcommon/include/common/logger_useful.h | 13 +++++++------ 6 files changed, 23 insertions(+), 14 deletions(-) diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 990cb582828..aa8edda58da 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -171,13 +171,13 @@ void TCPHandler::runImpl() send_exception_with_stack_trace = query_context->getSettingsRef().calculate_text_stack_trace; /// Should we send internal logs to client? - CLIENT_LOGS_LEVEL = query_context->getSettingsRef().send_logs_level.value; + const auto client_logs_level = query_context->getSettingsRef().send_logs_level.value; if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_LOGS - && CLIENT_LOGS_LEVEL != LogsLevel::none) + && client_logs_level != LogsLevel::none) { state.logs_queue = std::make_shared(); state.logs_queue->max_priority = Poco::Logger::parseLevel(query_context->getSettingsRef().send_logs_level.toString()); - CurrentThread::attachInternalTextLogsQueue(state.logs_queue); + CurrentThread::attachInternalTextLogsQueue(state.logs_queue, client_logs_level); } query_context->setExternalTablesInitializer([&global_settings, this] (Context & context) diff --git a/dbms/src/Common/CurrentThread.cpp b/dbms/src/Common/CurrentThread.cpp index 5186cec0c41..78dc57cb26b 100644 --- a/dbms/src/Common/CurrentThread.cpp +++ b/dbms/src/Common/CurrentThread.cpp @@ -60,11 +60,12 @@ void CurrentThread::updateProgressOut(const Progress & value) current_thread->progress_out.incrementPiecewiseAtomically(value); } -void CurrentThread::attachInternalTextLogsQueue(const std::shared_ptr & logs_queue) +void CurrentThread::attachInternalTextLogsQueue(const std::shared_ptr & logs_queue, + LogsLevel client_logs_level) { if (unlikely(!current_thread)) return; - current_thread->attachInternalTextLogsQueue(logs_queue); + current_thread->attachInternalTextLogsQueue(logs_queue, client_logs_level); } std::shared_ptr CurrentThread::getInternalTextLogsQueue() diff --git a/dbms/src/Common/CurrentThread.h b/dbms/src/Common/CurrentThread.h index 3c248ad903f..645c7f1d561 100644 --- a/dbms/src/Common/CurrentThread.h +++ b/dbms/src/Common/CurrentThread.h @@ -39,7 +39,8 @@ public: static ThreadGroupStatusPtr getGroup(); /// A logs queue used by TCPHandler to pass logs to a client - static void attachInternalTextLogsQueue(const std::shared_ptr & logs_queue); + static void attachInternalTextLogsQueue(const std::shared_ptr & logs_queue, + LogsLevel client_logs_level); static std::shared_ptr getInternalTextLogsQueue(); /// Makes system calls to update ProfileEvents that contain info from rusage and taskstats diff --git a/dbms/src/Common/ThreadStatus.cpp b/dbms/src/Common/ThreadStatus.cpp index c2e415ab363..bfc61868f7d 100644 --- a/dbms/src/Common/ThreadStatus.cpp +++ b/dbms/src/Common/ThreadStatus.cpp @@ -117,7 +117,8 @@ void ThreadStatus::assertState(const std::initializer_list & permitted_stat throw Exception(ss.str(), ErrorCodes::LOGICAL_ERROR); } -void ThreadStatus::attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue) +void ThreadStatus::attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue, + LogsLevel client_logs_level) { logs_queue_ptr = logs_queue; @@ -126,6 +127,7 @@ void ThreadStatus::attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & std::lock_guard lock(thread_group->mutex); thread_group->logs_queue_ptr = logs_queue; + thread_group->client_logs_level = client_logs_level; } } diff --git a/dbms/src/Common/ThreadStatus.h b/dbms/src/Common/ThreadStatus.h index 0d36024b15d..f853550abb4 100644 --- a/dbms/src/Common/ThreadStatus.h +++ b/dbms/src/Common/ThreadStatus.h @@ -4,6 +4,8 @@ #include #include +#include + #include #include @@ -62,6 +64,8 @@ public: UInt32 master_thread_number = 0; Int32 master_thread_os_id = -1; + LogsLevel client_logs_level = LogsLevel::none; + String query; }; @@ -130,7 +134,7 @@ public: return thread_state == Died ? nullptr : logs_queue_ptr.lock(); } - void attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue); + void attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue, LogsLevel client_logs_level); /// Sets query context for current thread and its thread group /// NOTE: query_context have to be alive until detachQuery() is called diff --git a/libs/libcommon/include/common/logger_useful.h b/libs/libcommon/include/common/logger_useful.h index a065646a4f8..10b3f7511f6 100644 --- a/libs/libcommon/include/common/logger_useful.h +++ b/libs/libcommon/include/common/logger_useful.h @@ -6,6 +6,7 @@ #include #include #include +#include #ifndef QUERY_PREVIEW_LENGTH #define QUERY_PREVIEW_LENGTH 160 @@ -13,37 +14,37 @@ using Poco::Logger; using DB::LogsLevel; +using DB::CurrentThread; -POCO_UNUSED static std::atomic CLIENT_LOGS_LEVEL = LogsLevel::none; /// Logs a message to a specified logger with that level. #define LOG_TRACE(logger, message) do { \ - if ((logger)->trace() || CLIENT_LOGS_LEVEL >= LogsLevel::trace) {\ + if ((logger)->trace() || CurrentThread::getGroup()->client_logs_level >= LogsLevel::trace) {\ std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ (logger)->trace(oss_internal_rare.str());}} while(false) #define LOG_DEBUG(logger, message) do { \ - if ((logger)->debug() || CLIENT_LOGS_LEVEL >= LogsLevel::debug) {\ + if ((logger)->debug() || CurrentThread::getGroup()->client_logs_level >= LogsLevel::debug) {\ std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ (logger)->debug(oss_internal_rare.str());}} while(false) #define LOG_INFO(logger, message) do { \ - if ((logger)->information() || CLIENT_LOGS_LEVEL >= LogsLevel::information) {\ + if ((logger)->information() || CurrentThread::getGroup()->client_logs_level >= LogsLevel::information) {\ std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ (logger)->information(oss_internal_rare.str());}} while(false) #define LOG_WARNING(logger, message) do { \ - if ((logger)->warning() || CLIENT_LOGS_LEVEL >= LogsLevel::warning) {\ + if ((logger)->warning() || CurrentThread::getGroup()->client_logs_level >= LogsLevel::warning) {\ std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ (logger)->warning(oss_internal_rare.str());}} while(false) #define LOG_ERROR(logger, message) do { \ - if ((logger)->error() || CLIENT_LOGS_LEVEL >= LogsLevel::error) {\ + if ((logger)->error() || CurrentThread::getGroup()->client_logs_level >= LogsLevel::error) {\ std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ (logger)->error(oss_internal_rare.str());}} while(false) From 865606b83cd398d47c6847fa0f444b1abc885450 Mon Sep 17 00:00:00 2001 From: NIKITA MIKHAILOV Date: Wed, 10 Jul 2019 15:19:17 +0300 Subject: [PATCH 07/64] bugfix client logs + some tests --- dbms/programs/server/TCPHandler.cpp | 15 ++-- dbms/src/Common/ThreadStatus.h | 7 +- .../Interpreters/InternalTextLogsQueue.cpp | 14 ++-- ...nd_logs_level_concurrent_queries.reference | 12 ++++ ...0965_send_logs_level_concurrent_queries.sh | 17 +++++ libs/libcommon/include/common/logger_useful.h | 69 ++++++++++++++----- 6 files changed, 105 insertions(+), 29 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.reference create mode 100755 dbms/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.sh diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index aa8edda58da..7014e55d99b 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -29,6 +29,11 @@ #include #include #include +<<<<<<< HEAD +======= + +#include +>>>>>>> aa54091152... bugfix client logs + some tests #include "TCPHandler.h" @@ -55,7 +60,7 @@ void TCPHandler::runImpl() ThreadStatus thread_status; connection_context = server.context(); - connection_context.makeSessionContext(); + connection_context.setSessionContext(connection_context); Settings global_settings = connection_context.getSettings(); @@ -171,13 +176,13 @@ void TCPHandler::runImpl() send_exception_with_stack_trace = query_context->getSettingsRef().calculate_text_stack_trace; /// Should we send internal logs to client? - const auto client_logs_level = query_context->getSettingsRef().send_logs_level.value; + const auto client_logs_level = query_context->getSettingsRef().send_logs_level; if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_LOGS - && client_logs_level != LogsLevel::none) + && client_logs_level.value != LogsLevel::none) { state.logs_queue = std::make_shared(); - state.logs_queue->max_priority = Poco::Logger::parseLevel(query_context->getSettingsRef().send_logs_level.toString()); - CurrentThread::attachInternalTextLogsQueue(state.logs_queue, client_logs_level); + state.logs_queue->max_priority = Poco::Logger::parseLevel(client_logs_level.toString()); + CurrentThread::attachInternalTextLogsQueue(state.logs_queue, client_logs_level.value); } query_context->setExternalTablesInitializer([&global_settings, this] (Context & context) diff --git a/dbms/src/Common/ThreadStatus.h b/dbms/src/Common/ThreadStatus.h index f853550abb4..08b6e2fa969 100644 --- a/dbms/src/Common/ThreadStatus.h +++ b/dbms/src/Common/ThreadStatus.h @@ -64,7 +64,7 @@ public: UInt32 master_thread_number = 0; Int32 master_thread_os_id = -1; - LogsLevel client_logs_level = LogsLevel::none; + std::atomic client_logs_level = LogsLevel::none; String query; }; @@ -134,7 +134,12 @@ public: return thread_state == Died ? nullptr : logs_queue_ptr.lock(); } +<<<<<<< HEAD void attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue, LogsLevel client_logs_level); +======= + void attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue, + LogsLevel client_logs_level); +>>>>>>> aa54091152... bugfix client logs + some tests /// Sets query context for current thread and its thread group /// NOTE: query_context have to be alive until detachQuery() is called diff --git a/dbms/src/Interpreters/InternalTextLogsQueue.cpp b/dbms/src/Interpreters/InternalTextLogsQueue.cpp index a27838a4203..6028514f11f 100644 --- a/dbms/src/Interpreters/InternalTextLogsQueue.cpp +++ b/dbms/src/Interpreters/InternalTextLogsQueue.cpp @@ -20,13 +20,13 @@ Block InternalTextLogsQueue::getSampleBlock() { return Block { {std::make_shared(), "event_time"}, - {std::make_shared(), "event_time_microseconds"}, - {std::make_shared(), "host_name"}, - {std::make_shared(), "query_id"}, - {std::make_shared(), "thread_number"}, - {std::make_shared(), "priority"}, - {std::make_shared(), "source"}, - {std::make_shared(), "text"} + {std::make_shared(), "event_time_microseconds"}, + {std::make_shared(), "host_name"}, + {std::make_shared(), "query_id"}, + {std::make_shared(), "thread_number"}, + {std::make_shared(), "priority"}, + {std::make_shared(), "source"}, + {std::make_shared(), "text"} }; } diff --git a/dbms/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.reference b/dbms/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.reference new file mode 100644 index 00000000000..45c133ba43b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.reference @@ -0,0 +1,12 @@ + + + + + + + + + + +***** + diff --git a/dbms/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.sh b/dbms/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.sh new file mode 100755 index 00000000000..97343c5acd7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +cp /dev/null 00965_send_logs_level_concurrent_queries_first.tmp +cp /dev/null 00965_send_logs_level_concurrent_queries_second.tmp + +clickhouse-client --send_logs_level="trace" --query="SELECT * from numbers(100000);" >> /dev/null 2>> 00965_send_logs_level_concurrent_queries_first.tmp & +clickhouse-client --send_logs_level="information" --query="SELECT * from numbers(100000);" >> /dev/null 2>> 00965_send_logs_level_concurrent_queries_second.tmp + +sleep 2 + +awk '{ print $8 }' 00965_send_logs_level_concurrent_queries_first.tmp +echo "*****" +awk '{ print $8 }' 00965_send_logs_level_concurrent_queries_second.tmp + diff --git a/libs/libcommon/include/common/logger_useful.h b/libs/libcommon/include/common/logger_useful.h index 10b3f7511f6..e4f5c101297 100644 --- a/libs/libcommon/include/common/logger_useful.h +++ b/libs/libcommon/include/common/logger_useful.h @@ -5,46 +5,83 @@ #include #include #include +#include #include #include +#include + #ifndef QUERY_PREVIEW_LENGTH #define QUERY_PREVIEW_LENGTH 160 #endif using Poco::Logger; +using Poco::Message; using DB::LogsLevel; using DB::CurrentThread; - /// Logs a message to a specified logger with that level. #define LOG_TRACE(logger, message) do { \ - if ((logger)->trace() || CurrentThread::getGroup()->client_logs_level >= LogsLevel::trace) {\ - std::stringstream oss_internal_rare; \ + const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && (CurrentThread::getGroup()->client_logs_level >= LogsLevel::trace); \ + if (is_clients_log) {\ + std::cerr << "CLIENTS LOG TRACE" << std::endl; \ + }\ + if ((logger)->trace() || is_clients_log) {\ + std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ - (logger)->trace(oss_internal_rare.str());}} while(false) + if (is_clients_log) {\ + (logger)->force_log(oss_internal_rare.str(), Message::PRIO_TRACE); \ + } else { \ + (logger)->trace(oss_internal_rare.str()); \ + }}} while(false) #define LOG_DEBUG(logger, message) do { \ - if ((logger)->debug() || CurrentThread::getGroup()->client_logs_level >= LogsLevel::debug) {\ - std::stringstream oss_internal_rare; \ + const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && (CurrentThread::getGroup()->client_logs_level >= LogsLevel::debug); \ + if (is_clients_log) {\ + std::cerr << "CLIENTS LOG DEBUG" << std::endl; \ + }\ + if ((logger)->debug() || is_clients_log) {\ + std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ - (logger)->debug(oss_internal_rare.str());}} while(false) + if (is_clients_log) {\ + (logger)->force_log(oss_internal_rare.str(), Message::PRIO_DEBUG); \ + } else { \ + (logger)->debug(oss_internal_rare.str()); \ + }}} while(false) #define LOG_INFO(logger, message) do { \ - if ((logger)->information() || CurrentThread::getGroup()->client_logs_level >= LogsLevel::information) {\ - std::stringstream oss_internal_rare; \ + const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && (CurrentThread::getGroup()->client_logs_level >= LogsLevel::information); \ + if (is_clients_log) {\ + std::cerr << "CLIENTS LOG INFORMATION" << std::endl; \ + }\ + if ((logger)->information() || is_clients_log) {\ + std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ - (logger)->information(oss_internal_rare.str());}} while(false) + if (is_clients_log) {\ + (logger)->force_log(oss_internal_rare.str(), Message::PRIO_INFORMATION); \ + } else { \ + (logger)->information(oss_internal_rare.str()); \ + }}} while(false) #define LOG_WARNING(logger, message) do { \ - if ((logger)->warning() || CurrentThread::getGroup()->client_logs_level >= LogsLevel::warning) {\ - std::stringstream oss_internal_rare; \ + const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && (CurrentThread::getGroup()->client_logs_level >= LogsLevel::warning); \ + if ((logger)->warning() || is_clients_log) {\ + std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ - (logger)->warning(oss_internal_rare.str());}} while(false) + if (is_clients_log) {\ + (logger)->force_log(oss_internal_rare.str(), Message::PRIO_WARNING); \ + } else { \ + (logger)->warning(oss_internal_rare.str()); \ + }}} while(false) #define LOG_ERROR(logger, message) do { \ - if ((logger)->error() || CurrentThread::getGroup()->client_logs_level >= LogsLevel::error) {\ - std::stringstream oss_internal_rare; \ + const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && (CurrentThread::getGroup()->client_logs_level >= LogsLevel::error); \ + if ((logger)->error() || is_clients_log) {\ + std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ - (logger)->error(oss_internal_rare.str());}} while(false) + if (is_clients_log) {\ + (logger)->force_log(oss_internal_rare.str(), Message::PRIO_ERROR); \ + } else { \ + (logger)->error(oss_internal_rare.str()); \ + }}} while(false) From e05f5388bc628800e396eb5b99325a54b6d0ab43 Mon Sep 17 00:00:00 2001 From: NIKITA MIKHAILOV Date: Wed, 10 Jul 2019 16:27:16 +0300 Subject: [PATCH 08/64] clean after cherry-pick --- dbms/programs/server/TCPHandler.cpp | 5 ----- dbms/src/Common/ThreadStatus.h | 4 ---- 2 files changed, 9 deletions(-) diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 7014e55d99b..91dc44d4b94 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -29,11 +29,6 @@ #include #include #include -<<<<<<< HEAD -======= - -#include ->>>>>>> aa54091152... bugfix client logs + some tests #include "TCPHandler.h" diff --git a/dbms/src/Common/ThreadStatus.h b/dbms/src/Common/ThreadStatus.h index 08b6e2fa969..dbbbfca4988 100644 --- a/dbms/src/Common/ThreadStatus.h +++ b/dbms/src/Common/ThreadStatus.h @@ -134,12 +134,8 @@ public: return thread_state == Died ? nullptr : logs_queue_ptr.lock(); } -<<<<<<< HEAD - void attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue, LogsLevel client_logs_level); -======= void attachInternalTextLogsQueue(const InternalTextLogsQueuePtr & logs_queue, LogsLevel client_logs_level); ->>>>>>> aa54091152... bugfix client logs + some tests /// Sets query context for current thread and its thread group /// NOTE: query_context have to be alive until detachQuery() is called From 90494589e4b51bad45fefb4a09a29d4120cf0586 Mon Sep 17 00:00:00 2001 From: NIKITA MIKHAILOV Date: Wed, 10 Jul 2019 16:32:12 +0300 Subject: [PATCH 09/64] remove useless stderr --- libs/libcommon/include/common/logger_useful.h | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/libs/libcommon/include/common/logger_useful.h b/libs/libcommon/include/common/logger_useful.h index e4f5c101297..0878c589397 100644 --- a/libs/libcommon/include/common/logger_useful.h +++ b/libs/libcommon/include/common/logger_useful.h @@ -9,8 +9,6 @@ #include #include -#include - #ifndef QUERY_PREVIEW_LENGTH #define QUERY_PREVIEW_LENGTH 160 #endif @@ -24,9 +22,6 @@ using DB::CurrentThread; #define LOG_TRACE(logger, message) do { \ const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && (CurrentThread::getGroup()->client_logs_level >= LogsLevel::trace); \ - if (is_clients_log) {\ - std::cerr << "CLIENTS LOG TRACE" << std::endl; \ - }\ if ((logger)->trace() || is_clients_log) {\ std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ @@ -38,9 +33,6 @@ using DB::CurrentThread; #define LOG_DEBUG(logger, message) do { \ const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && (CurrentThread::getGroup()->client_logs_level >= LogsLevel::debug); \ - if (is_clients_log) {\ - std::cerr << "CLIENTS LOG DEBUG" << std::endl; \ - }\ if ((logger)->debug() || is_clients_log) {\ std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ @@ -52,9 +44,6 @@ using DB::CurrentThread; #define LOG_INFO(logger, message) do { \ const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && (CurrentThread::getGroup()->client_logs_level >= LogsLevel::information); \ - if (is_clients_log) {\ - std::cerr << "CLIENTS LOG INFORMATION" << std::endl; \ - }\ if ((logger)->information() || is_clients_log) {\ std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ From 7acd275b7030ec82eef6a5d4ba87b2977ee2ab31 Mon Sep 17 00:00:00 2001 From: NIKITA MIKHAILOV Date: Wed, 10 Jul 2019 18:49:35 +0300 Subject: [PATCH 10/64] add my poco --- contrib/poco | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/poco b/contrib/poco index ece721f1085..4ea95983f9a 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit ece721f1085e3894cb5286e8560af84cd1445326 +Subproject commit 4ea95983f9a92c9e5a21b6e782b4b3de330230ca From 18a13a03dd7689540d7148c9dca1141c69e856b1 Mon Sep 17 00:00:00 2001 From: chertus Date: Wed, 10 Jul 2019 21:12:50 +0300 Subject: [PATCH 11/64] memory tracked new/delete concept --- contrib/jemalloc-cmake/CMakeLists.txt | 1 - dbms/src/Common/CurrentThread.cpp | 6 +++++ dbms/src/Common/CurrentThread.h | 1 + dbms/src/Common/MemoryTracker.cpp | 28 ++++++++++++++++++---- dbms/src/Common/MemoryTracker.h | 6 ++++- dbms/src/Common/ThreadStatus.cpp | 5 ++++ dbms/src/Common/ThreadStatus.h | 2 ++ dbms/src/Common/new_delete.cpp | 34 +++++++++++++++++++++++++++ 8 files changed, 77 insertions(+), 6 deletions(-) create mode 100644 dbms/src/Common/new_delete.cpp diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 4840197c2fd..47f057c0559 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -15,7 +15,6 @@ ${JEMALLOC_SOURCE_DIR}/src/extent_mmap.c ${JEMALLOC_SOURCE_DIR}/src/hash.c ${JEMALLOC_SOURCE_DIR}/src/hook.c ${JEMALLOC_SOURCE_DIR}/src/jemalloc.c -${JEMALLOC_SOURCE_DIR}/src/jemalloc_cpp.cpp ${JEMALLOC_SOURCE_DIR}/src/large.c ${JEMALLOC_SOURCE_DIR}/src/log.c ${JEMALLOC_SOURCE_DIR}/src/malloc_io.c diff --git a/dbms/src/Common/CurrentThread.cpp b/dbms/src/Common/CurrentThread.cpp index 5186cec0c41..5ef8d60309c 100644 --- a/dbms/src/Common/CurrentThread.cpp +++ b/dbms/src/Common/CurrentThread.cpp @@ -46,6 +46,12 @@ MemoryTracker * CurrentThread::getMemoryTracker() return ¤t_thread->memory_tracker; } +Int64 & CurrentThread::getUntrackedMemory() +{ + /// It assumes that (current_thread != nullptr) is already checked with getMemoryTracker() + return current_thread->untracked_memory; +} + void CurrentThread::updateProgressIn(const Progress & value) { if (unlikely(!current_thread)) diff --git a/dbms/src/Common/CurrentThread.h b/dbms/src/Common/CurrentThread.h index 3c248ad903f..cfab52dcdf5 100644 --- a/dbms/src/Common/CurrentThread.h +++ b/dbms/src/Common/CurrentThread.h @@ -47,6 +47,7 @@ public: static ProfileEvents::Counters & getProfileEvents(); static MemoryTracker * getMemoryTracker(); + static Int64 & getUntrackedMemory(); /// Update read and write rows (bytes) statistics (used in system.query_thread_log) static void updateProgressIn(const Progress & value); diff --git a/dbms/src/Common/MemoryTracker.cpp b/dbms/src/Common/MemoryTracker.cpp index bc324be4904..2d9380707d7 100644 --- a/dbms/src/Common/MemoryTracker.cpp +++ b/dbms/src/Common/MemoryTracker.cpp @@ -17,6 +17,7 @@ namespace DB static constexpr size_t log_peak_memory_usage_every = 1ULL << 30; +static constexpr Int64 untracked_memory_limit = 4 * 1024 * 1024; MemoryTracker::~MemoryTracker() @@ -191,19 +192,38 @@ namespace CurrentMemoryTracker void alloc(Int64 size) { if (auto memory_tracker = DB::CurrentThread::getMemoryTracker()) - memory_tracker->alloc(size); + { + Int64 & untracked = DB::CurrentThread::getUntrackedMemory(); + untracked += size; + if (untracked > untracked_memory_limit) + { + memory_tracker->alloc(untracked); + untracked = 0; + } + } } void realloc(Int64 old_size, Int64 new_size) { - if (auto memory_tracker = DB::CurrentThread::getMemoryTracker()) - memory_tracker->alloc(new_size - old_size); + Int64 addition = new_size - old_size; + if (addition > 0) + alloc(addition); + else + free(-addition); } void free(Int64 size) { if (auto memory_tracker = DB::CurrentThread::getMemoryTracker()) - memory_tracker->free(size); + { + Int64 & untracked = DB::CurrentThread::getUntrackedMemory(); + untracked -= size; + if (untracked < -untracked_memory_limit) + { + memory_tracker->free(-untracked); + untracked = 0; + } + } } } diff --git a/dbms/src/Common/MemoryTracker.h b/dbms/src/Common/MemoryTracker.h index 9f439c7550c..4ce0ac262fa 100644 --- a/dbms/src/Common/MemoryTracker.h +++ b/dbms/src/Common/MemoryTracker.h @@ -45,7 +45,11 @@ public: void realloc(Int64 old_size, Int64 new_size) { - alloc(new_size - old_size); + Int64 addition = new_size - old_size; + if (addition > 0) + alloc(addition); + else + free(-addition); } /** This function should be called after memory deallocation. diff --git a/dbms/src/Common/ThreadStatus.cpp b/dbms/src/Common/ThreadStatus.cpp index c2e415ab363..2e57a5237b4 100644 --- a/dbms/src/Common/ThreadStatus.cpp +++ b/dbms/src/Common/ThreadStatus.cpp @@ -50,6 +50,11 @@ ThreadStatus::ThreadStatus() ThreadStatus::~ThreadStatus() { + if (untracked_memory > 0) + memory_tracker.alloc(untracked_memory); + else + memory_tracker.free(-untracked_memory); + if (deleter) deleter(); current_thread = nullptr; diff --git a/dbms/src/Common/ThreadStatus.h b/dbms/src/Common/ThreadStatus.h index 0d36024b15d..6ea6f5c19fb 100644 --- a/dbms/src/Common/ThreadStatus.h +++ b/dbms/src/Common/ThreadStatus.h @@ -92,6 +92,8 @@ public: /// TODO: merge them into common entity ProfileEvents::Counters performance_counters{VariableContext::Thread}; MemoryTracker memory_tracker{VariableContext::Thread}; + /// Small amount of untracked memory (per thread atomic-less counter) + Int64 untracked_memory = 0; /// Statistics of read and write rows/bytes Progress progress_in; diff --git a/dbms/src/Common/new_delete.cpp b/dbms/src/Common/new_delete.cpp new file mode 100644 index 00000000000..faf638e713b --- /dev/null +++ b/dbms/src/Common/new_delete.cpp @@ -0,0 +1,34 @@ +#include + +#include +#include + +/// Replace default new/delete with memory tracking versions. +/// @sa https://en.cppreference.com/w/cpp/memory/new/operator_new +/// https://en.cppreference.com/w/cpp/memory/new/operator_delete +#if 1 + +void * operator new (std::size_t size) +{ + CurrentMemoryTracker::alloc(size); + + auto * ptr = malloc(size); + if (likely(ptr != nullptr)) + return ptr; + + CurrentMemoryTracker::free(size); + + /// @note no std::get_new_handler logic implemented + std::__throw_bad_alloc(); +} + +/// Called instead of 'delete(void * ptr)' if a user-defined replacement is provided +void operator delete (void * ptr, std::size_t size) noexcept +{ + CurrentMemoryTracker::free(size); + + if (likely(ptr != nullptr)) + free(ptr); +} + +#endif From 9739ac13e43c5d7148b5495d60f661c89296ccd1 Mon Sep 17 00:00:00 2001 From: chertus Date: Thu, 11 Jul 2019 18:40:55 +0300 Subject: [PATCH 12/64] move new/delete overloads to dbms/src/Common --- dbms/CMakeLists.txt | 1 + dbms/src/Common/MemoryTracker.cpp | 10 ++- dbms/src/Common/config.h.in | 1 + dbms/src/Common/new_delete.cpp | 93 ++++++++++++++++++++----- libs/libcommon/CMakeLists.txt | 1 + libs/libcommon/include/common/memory.h | 96 ++++++++++++++++++++++++++ libs/libcommon/src/jemalloc_cpp.cpp | 37 ++++++++++ 7 files changed, 221 insertions(+), 18 deletions(-) create mode 100644 libs/libcommon/include/common/memory.h create mode 100644 libs/libcommon/src/jemalloc_cpp.cpp diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index bcbe435569f..260040579df 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -365,6 +365,7 @@ endif() if (USE_JEMALLOC) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # used in Interpreters/AsynchronousMetrics.cpp + target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # new_delete.cpp endif () target_include_directories (dbms PUBLIC ${DBMS_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/Formats/include) diff --git a/dbms/src/Common/MemoryTracker.cpp b/dbms/src/Common/MemoryTracker.cpp index 2d9380707d7..a7af62636a0 100644 --- a/dbms/src/Common/MemoryTracker.cpp +++ b/dbms/src/Common/MemoryTracker.cpp @@ -85,7 +85,7 @@ void MemoryTracker::alloc(Int64 size) if (unlikely(fault_probability && drand48() < fault_probability)) { free(size); - +#if 0 std::stringstream message; message << "Memory tracker"; if (description) @@ -95,12 +95,15 @@ void MemoryTracker::alloc(Int64 size) << ", maximum: " << formatReadableSizeWithBinarySuffix(current_limit); throw DB::Exception(message.str(), DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED); +#else + throw DB::Exception("Memory limit exceeded", DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED); +#endif } if (unlikely(current_limit && will_be > current_limit)) { free(size); - +#if 0 std::stringstream message; message << "Memory limit"; if (description) @@ -110,6 +113,9 @@ void MemoryTracker::alloc(Int64 size) << ", maximum: " << formatReadableSizeWithBinarySuffix(current_limit); throw DB::Exception(message.str(), DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED); +#else + throw DB::Exception("Memory limit exceeded", DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED); +#endif } auto peak_old = peak.load(std::memory_order_relaxed); diff --git a/dbms/src/Common/config.h.in b/dbms/src/Common/config.h.in index 9b38dd9fc04..556eeebec58 100644 --- a/dbms/src/Common/config.h.in +++ b/dbms/src/Common/config.h.in @@ -9,5 +9,6 @@ #cmakedefine01 USE_CPUINFO #cmakedefine01 USE_BROTLI #cmakedefine01 USE_MIMALLOC +#cmakedefine01 USE_JEMALLOC #cmakedefine01 CLICKHOUSE_SPLIT_BINARY diff --git a/dbms/src/Common/new_delete.cpp b/dbms/src/Common/new_delete.cpp index faf638e713b..c0e6633c2c5 100644 --- a/dbms/src/Common/new_delete.cpp +++ b/dbms/src/Common/new_delete.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include /// Replace default new/delete with memory tracking versions. @@ -8,27 +8,88 @@ /// https://en.cppreference.com/w/cpp/memory/new/operator_delete #if 1 -void * operator new (std::size_t size) +/// new + +void * operator new(std::size_t size) { CurrentMemoryTracker::alloc(size); - - auto * ptr = malloc(size); - if (likely(ptr != nullptr)) - return ptr; - - CurrentMemoryTracker::free(size); - - /// @note no std::get_new_handler logic implemented - std::__throw_bad_alloc(); + return Memory::newImpl(size); } -/// Called instead of 'delete(void * ptr)' if a user-defined replacement is provided -void operator delete (void * ptr, std::size_t size) noexcept +void * operator new[](std::size_t size) +{ + CurrentMemoryTracker::alloc(size); + return Memory::newImpl(size); +} + +void * operator new(std::size_t size, const std::nothrow_t &) noexcept +{ + CurrentMemoryTracker::alloc(size); + return Memory::newNoExept(size); +} + +void * operator new[](std::size_t size, const std::nothrow_t &) noexcept +{ + CurrentMemoryTracker::alloc(size); + return Memory::newNoExept(size); +} + +/// delete + +#if 0 +void operator delete(void * ptr) noexcept +{ + Memory::deleteImpl(ptr); +} + +void operator delete[](void * ptr) noexcept +{ + Memory::deleteImpl(ptr); +} + + +void operator delete(void * ptr, const std::nothrow_t &) noexcept +{ + Memory::deleteImpl(ptr); +} + +void operator delete[](void * ptr, const std::nothrow_t &) noexcept +{ + Memory::deleteImpl(ptr); +} +#endif + +void operator delete(void * ptr, std::size_t size) noexcept { CurrentMemoryTracker::free(size); - - if (likely(ptr != nullptr)) - free(ptr); + Memory::deleteSized(ptr, size); } +void operator delete[](void * ptr, std::size_t size) noexcept +{ + CurrentMemoryTracker::free(size); + Memory::deleteSized(ptr, size); +} + +#else + +/// new + +void * operator new(std::size_t size) { return Memory::newImpl(size); } +void * operator new[](std::size_t size) { return Memory::newImpl(size); } + +void * operator new(std::size_t size, const std::nothrow_t &) noexcept { return Memory::newNoExept(size); } +void * operator new[](std::size_t size, const std::nothrow_t &) noexcept { return Memory::newNoExept(size); } + +/// delete + +void operator delete(void * ptr) noexcept { Memory::deleteImpl(ptr); } +void operator delete[](void * ptr) noexcept { Memory::deleteImpl(ptr); } + +void operator delete(void * ptr, const std::nothrow_t &) noexcept { Memory::deleteImpl(ptr); } +void operator delete[](void * ptr, const std::nothrow_t &) noexcept { Memory::deleteImpl(ptr); } + +void operator delete(void * ptr, std::size_t size) noexcept { Memory::deleteSized(ptr, size); } +void operator delete[](void * ptr, std::size_t size) noexcept { Memory::deleteSized(ptr, size); } + #endif diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index e3fac95694a..914fc8ba4da 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -23,6 +23,7 @@ add_library(common src/getThreadNumber.cpp src/argsToConfig.cpp src/StackTrace.cpp + src/jemalloc_cpp.cpp include/common/SimpleCache.h include/common/StackTrace.h diff --git a/libs/libcommon/include/common/memory.h b/libs/libcommon/include/common/memory.h new file mode 100644 index 00000000000..4a93f27a4a2 --- /dev/null +++ b/libs/libcommon/include/common/memory.h @@ -0,0 +1,96 @@ +#pragma once + +#if __has_include() +#include +#endif + +#if USE_JEMALLOC + +#include +#include +#include + +#if defined(_MSC_VER) + #define ALWAYS_INLINE __forceinline + #define NO_INLINE static __declspec(noinline) +#else + #define ALWAYS_INLINE __attribute__((__always_inline__)) + #define NO_INLINE __attribute__((__noinline__)) +#endif + +namespace JeMalloc +{ + +void * handleOOM(std::size_t size, bool nothrow); + +ALWAYS_INLINE inline void * newImpl(std::size_t size) +{ + void * ptr = je_malloc(size); + if (likely(ptr != nullptr)) + return ptr; + + return handleOOM(size, false); +} + +ALWAYS_INLINE inline void * newNoExept(std::size_t size) noexcept +{ + void * ptr = je_malloc(size); + if (likely(ptr != nullptr)) + return ptr; + + return handleOOM(size, true); +} + +ALWAYS_INLINE inline void deleteImpl(void * ptr) noexcept +{ + je_free(ptr); +} + +ALWAYS_INLINE inline void deleteSized(void * ptr, std::size_t size) noexcept +{ + if (unlikely(ptr == nullptr)) + return; + + je_sdallocx(ptr, size, 0); +} + +} + +namespace Memory +{ + using namespace JeMalloc; +} + +#else + +namespace Memory +{ + +ALWAYS_INLINE inline void * newImpl(std::size_t size) +{ + auto * ptr = malloc(size); + if (likely(ptr != nullptr)) + return ptr; + + /// @note no std::get_new_handler logic implemented + std::__throw_bad_alloc(); +} + +ALWAYS_INLINE inline void * newNoExept(std::size_t size) noexcept +{ + return malloc(size); +} + +ALWAYS_INLINE inline void deleteImpl(void * ptr) noexcept +{ + free(ptr); +} + +ALWAYS_INLINE inline void deleteSized(void * ptr, std::size_t size) noexcept +{ + free(ptr); +} + +} + +#endif diff --git a/libs/libcommon/src/jemalloc_cpp.cpp b/libs/libcommon/src/jemalloc_cpp.cpp new file mode 100644 index 00000000000..72b3b1ef856 --- /dev/null +++ b/libs/libcommon/src/jemalloc_cpp.cpp @@ -0,0 +1,37 @@ +#include + +#if USE_JEMALLOC + +namespace JeMalloc +{ + +void * handleOOM(std::size_t size, bool nothrow) +{ + void * ptr = nullptr; + + while (ptr == nullptr) + { + std::new_handler handler = std::get_new_handler(); + if (handler == nullptr) + break; + + try + { + handler(); + } + catch (const std::bad_alloc &) + { + break; + } + + ptr = je_malloc(size); + } + + if (ptr == nullptr && !nothrow) + std::__throw_bad_alloc(); + return ptr; +} + +} + +#endif // USE_JEMALLOC From 9bd42366f0b50b74b1790983a5cf272bf3166d88 Mon Sep 17 00:00:00 2001 From: chertus Date: Fri, 12 Jul 2019 17:41:59 +0300 Subject: [PATCH 13/64] build fixes --- dbms/CMakeLists.txt | 6 ++++++ dbms/src/Common/new_delete.cpp | 21 ++++++++------------- libs/libcommon/include/common/memory.h | 20 ++++++++++---------- libs/libcommon/src/jemalloc_cpp.cpp | 2 +- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 260040579df..79cfd602a7c 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -90,6 +90,12 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") add_definitions ("-fno-tree-loop-distribute-patterns") endif () +if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR + CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # Enable C++14 sized global deallocation functions. It should be anabled by setting -std=c++14 + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation") +endif () + add_subdirectory (src) set(dbms_headers) diff --git a/dbms/src/Common/new_delete.cpp b/dbms/src/Common/new_delete.cpp index c0e6633c2c5..b3e349ffc1a 100644 --- a/dbms/src/Common/new_delete.cpp +++ b/dbms/src/Common/new_delete.cpp @@ -36,7 +36,14 @@ void * operator new[](std::size_t size, const std::nothrow_t &) noexcept /// delete -#if 0 +/// C++17 std 21.6.2.1 (11) +/// If a function without a size parameter is defined, the program should also define the corresponding function with a size parameter. +/// If a function with a size parameter is defined, the program shall also define the corresponding version without the size parameter. + +/// cppreference: +/// It's unspecified whether size-aware or size-unaware version is called when deleting objects of +/// incomplete type and arrays of non-class and trivially-destructible class types. + void operator delete(void * ptr) noexcept { Memory::deleteImpl(ptr); @@ -47,18 +54,6 @@ void operator delete[](void * ptr) noexcept Memory::deleteImpl(ptr); } - -void operator delete(void * ptr, const std::nothrow_t &) noexcept -{ - Memory::deleteImpl(ptr); -} - -void operator delete[](void * ptr, const std::nothrow_t &) noexcept -{ - Memory::deleteImpl(ptr); -} -#endif - void operator delete(void * ptr, std::size_t size) noexcept { CurrentMemoryTracker::free(size); diff --git a/libs/libcommon/include/common/memory.h b/libs/libcommon/include/common/memory.h index 4a93f27a4a2..bd4c3ac578c 100644 --- a/libs/libcommon/include/common/memory.h +++ b/libs/libcommon/include/common/memory.h @@ -14,7 +14,7 @@ #define ALWAYS_INLINE __forceinline #define NO_INLINE static __declspec(noinline) #else - #define ALWAYS_INLINE __attribute__((__always_inline__)) + #define ALWAYS_INLINE inline __attribute__((__always_inline__)) #define NO_INLINE __attribute__((__noinline__)) #endif @@ -23,7 +23,7 @@ namespace JeMalloc void * handleOOM(std::size_t size, bool nothrow); -ALWAYS_INLINE inline void * newImpl(std::size_t size) +ALWAYS_INLINE void * newImpl(std::size_t size) { void * ptr = je_malloc(size); if (likely(ptr != nullptr)) @@ -32,7 +32,7 @@ ALWAYS_INLINE inline void * newImpl(std::size_t size) return handleOOM(size, false); } -ALWAYS_INLINE inline void * newNoExept(std::size_t size) noexcept +ALWAYS_INLINE void * newNoExept(std::size_t size) noexcept { void * ptr = je_malloc(size); if (likely(ptr != nullptr)) @@ -41,12 +41,12 @@ ALWAYS_INLINE inline void * newNoExept(std::size_t size) noexcept return handleOOM(size, true); } -ALWAYS_INLINE inline void deleteImpl(void * ptr) noexcept +ALWAYS_INLINE void deleteImpl(void * ptr) noexcept { je_free(ptr); } -ALWAYS_INLINE inline void deleteSized(void * ptr, std::size_t size) noexcept +ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size) noexcept { if (unlikely(ptr == nullptr)) return; @@ -66,27 +66,27 @@ namespace Memory namespace Memory { -ALWAYS_INLINE inline void * newImpl(std::size_t size) +ALWAYS_INLINE void * newImpl(std::size_t size) { auto * ptr = malloc(size); if (likely(ptr != nullptr)) return ptr; /// @note no std::get_new_handler logic implemented - std::__throw_bad_alloc(); + throw std::bad_alloc{}; } -ALWAYS_INLINE inline void * newNoExept(std::size_t size) noexcept +ALWAYS_INLINE void * newNoExept(std::size_t size) noexcept { return malloc(size); } -ALWAYS_INLINE inline void deleteImpl(void * ptr) noexcept +ALWAYS_INLINE void deleteImpl(void * ptr) noexcept { free(ptr); } -ALWAYS_INLINE inline void deleteSized(void * ptr, std::size_t size) noexcept +ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size) noexcept { free(ptr); } diff --git a/libs/libcommon/src/jemalloc_cpp.cpp b/libs/libcommon/src/jemalloc_cpp.cpp index 72b3b1ef856..fbf074a0fb5 100644 --- a/libs/libcommon/src/jemalloc_cpp.cpp +++ b/libs/libcommon/src/jemalloc_cpp.cpp @@ -28,7 +28,7 @@ void * handleOOM(std::size_t size, bool nothrow) } if (ptr == nullptr && !nothrow) - std::__throw_bad_alloc(); + throw std::bad_alloc{}; return ptr; } From a4bbb391953d79d6096c26b61fb10579912d3bc4 Mon Sep 17 00:00:00 2001 From: chertus Date: Fri, 12 Jul 2019 20:06:02 +0300 Subject: [PATCH 14/64] fix crash cause of recursion in allocs and memory tracking --- dbms/src/Common/MemoryTracker.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dbms/src/Common/MemoryTracker.cpp b/dbms/src/Common/MemoryTracker.cpp index a7af62636a0..380dee75748 100644 --- a/dbms/src/Common/MemoryTracker.cpp +++ b/dbms/src/Common/MemoryTracker.cpp @@ -85,7 +85,10 @@ void MemoryTracker::alloc(Int64 size) if (unlikely(fault_probability && drand48() < fault_probability)) { free(size); -#if 0 + + /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc + auto untrack_lock = blocker.cancel(); + std::stringstream message; message << "Memory tracker"; if (description) @@ -95,15 +98,15 @@ void MemoryTracker::alloc(Int64 size) << ", maximum: " << formatReadableSizeWithBinarySuffix(current_limit); throw DB::Exception(message.str(), DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED); -#else - throw DB::Exception("Memory limit exceeded", DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED); -#endif } if (unlikely(current_limit && will_be > current_limit)) { free(size); -#if 0 + + /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc + auto untrack_lock = blocker.cancel(); + std::stringstream message; message << "Memory limit"; if (description) @@ -113,9 +116,6 @@ void MemoryTracker::alloc(Int64 size) << ", maximum: " << formatReadableSizeWithBinarySuffix(current_limit); throw DB::Exception(message.str(), DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED); -#else - throw DB::Exception("Memory limit exceeded", DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED); -#endif } auto peak_old = peak.load(std::memory_order_relaxed); From 8c715d9b91dde9b37e0c8697ba7b714574ff3f1e Mon Sep 17 00:00:00 2001 From: chertus Date: Fri, 12 Jul 2019 20:22:20 +0300 Subject: [PATCH 15/64] minor fix in cmake-files --- CMakeLists.txt | 5 +++++ dbms/CMakeLists.txt | 6 ------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c270cf7feb7..955550f0061 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -182,6 +182,11 @@ else () set (CXX_FLAGS_INTERNAL_COMPILER "-std=c++1z") endif () +if (COMPILER_GCC OR COMPILER_CLANG) + # Enable C++14 sized global deallocation functions. It should be enabled by setting -std=c++14 but I'm not sure. + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation") +endif () + option(WITH_COVERAGE "Build with coverage." 0) if(WITH_COVERAGE AND COMPILER_CLANG) set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-instr-generate -fcoverage-mapping") diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 5af4449a5f4..97807a5c9d4 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -90,12 +90,6 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") add_definitions ("-fno-tree-loop-distribute-patterns") endif () -if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR - CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - # Enable C++14 sized global deallocation functions. It should be anabled by setting -std=c++14 - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation") -endif () - add_subdirectory (src) set(dbms_headers) From 81d8597bb9062e0d747ae02675eda2c98696e594 Mon Sep 17 00:00:00 2001 From: chertus Date: Mon, 15 Jul 2019 16:19:56 +0300 Subject: [PATCH 16/64] memory tracking for size-unaware deletes with jemalloc --- dbms/src/Common/new_delete.cpp | 53 +++++++++++++++---- ...0877_memory_limit_for_new_delete.reference | 0 .../00877_memory_limit_for_new_delete.sql | 7 +++ 3 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00877_memory_limit_for_new_delete.reference create mode 100644 dbms/tests/queries/0_stateless/00877_memory_limit_for_new_delete.sql diff --git a/dbms/src/Common/new_delete.cpp b/dbms/src/Common/new_delete.cpp index b3e349ffc1a..f790eef85cf 100644 --- a/dbms/src/Common/new_delete.cpp +++ b/dbms/src/Common/new_delete.cpp @@ -8,30 +8,61 @@ /// https://en.cppreference.com/w/cpp/memory/new/operator_delete #if 1 +namespace Memory +{ + +ALWAYS_INLINE void * trackMemory(void * ptr [[maybe_unused]], std::size_t size [[maybe_unused]]) noexcept +{ +#ifdef USE_JEMALLOC + if (likely(ptr != nullptr)) + CurrentMemoryTracker::alloc(sallocx(ptr, 0)); +#else + CurrentMemoryTracker::alloc(size); +#endif + + return ptr; +} + +ALWAYS_INLINE void untrackMemory(void * ptr [[maybe_unused]]) noexcept +{ +#ifdef USE_JEMALLOC + if (likely(ptr != nullptr)) + CurrentMemoryTracker::free(sallocx(ptr, 0)); +#endif +} + +ALWAYS_INLINE void untrackMemory(void * ptr [[maybe_unused]], std::size_t size [[maybe_unused]]) noexcept +{ +#ifdef USE_JEMALLOC + if (likely(ptr != nullptr)) + CurrentMemoryTracker::free(sallocx(ptr, 0)); +#else + CurrentMemoryTracker::free(size); +#endif +} + +} + /// new void * operator new(std::size_t size) { - CurrentMemoryTracker::alloc(size); - return Memory::newImpl(size); + return Memory::trackMemory(Memory::newImpl(size), size); } void * operator new[](std::size_t size) { - CurrentMemoryTracker::alloc(size); - return Memory::newImpl(size); + return Memory::trackMemory(Memory::newImpl(size), size); } void * operator new(std::size_t size, const std::nothrow_t &) noexcept { - CurrentMemoryTracker::alloc(size); - return Memory::newNoExept(size); + return Memory::trackMemory(Memory::newNoExept(size), size); } void * operator new[](std::size_t size, const std::nothrow_t &) noexcept { - CurrentMemoryTracker::alloc(size); - return Memory::newNoExept(size); + return Memory::trackMemory(Memory::newNoExept(size), size); } /// delete @@ -46,23 +77,25 @@ void * operator new[](std::size_t size, const std::nothrow_t &) noexcept void operator delete(void * ptr) noexcept { + Memory::untrackMemory(ptr); Memory::deleteImpl(ptr); } void operator delete[](void * ptr) noexcept { + Memory::untrackMemory(ptr); Memory::deleteImpl(ptr); } void operator delete(void * ptr, std::size_t size) noexcept { - CurrentMemoryTracker::free(size); + Memory::untrackMemory(ptr, size); Memory::deleteSized(ptr, size); } void operator delete[](void * ptr, std::size_t size) noexcept { - CurrentMemoryTracker::free(size); + Memory::untrackMemory(ptr, size); Memory::deleteSized(ptr, size); } diff --git a/dbms/tests/queries/0_stateless/00877_memory_limit_for_new_delete.reference b/dbms/tests/queries/0_stateless/00877_memory_limit_for_new_delete.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00877_memory_limit_for_new_delete.sql b/dbms/tests/queries/0_stateless/00877_memory_limit_for_new_delete.sql new file mode 100644 index 00000000000..111104bb06e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00877_memory_limit_for_new_delete.sql @@ -0,0 +1,7 @@ +SET max_memory_usage = 1000000000; + +SELECT sum(ignore(*)) FROM ( + SELECT number, argMax(number, (number, toFixedString(toString(number), 1024))) + FROM numbers(1000000) + GROUP BY number +) -- { serverError 241 } From d192df13883192750c7d722587c2796679dca59c Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Mon, 15 Jul 2019 18:25:32 +0300 Subject: [PATCH 17/64] poco patch define added --- ...965_pocopatch_logs_level_bugfix.reference} | 0 ...h => 00965_pocopatch_logs_level_bugfix.sh} | 0 ...d_logs_level_concurrent_queries.reference} | 0 ...tch_send_logs_level_concurrent_queries.sh} | 0 libs/libcommon/include/common/logger_useful.h | 37 +++++++++++++++++++ 5 files changed, 37 insertions(+) rename dbms/tests/queries/0_stateless/{00965_logs_level_bugfix.reference => 00965_pocopatch_logs_level_bugfix.reference} (100%) rename dbms/tests/queries/0_stateless/{00965_logs_level_bugfix.sh => 00965_pocopatch_logs_level_bugfix.sh} (100%) rename dbms/tests/queries/0_stateless/{00965_send_logs_level_concurrent_queries.reference => 00965_pocopatch_send_logs_level_concurrent_queries.reference} (100%) rename dbms/tests/queries/0_stateless/{00965_send_logs_level_concurrent_queries.sh => 00965_pocopatch_send_logs_level_concurrent_queries.sh} (100%) diff --git a/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.reference b/dbms/tests/queries/0_stateless/00965_pocopatch_logs_level_bugfix.reference similarity index 100% rename from dbms/tests/queries/0_stateless/00965_logs_level_bugfix.reference rename to dbms/tests/queries/0_stateless/00965_pocopatch_logs_level_bugfix.reference diff --git a/dbms/tests/queries/0_stateless/00965_logs_level_bugfix.sh b/dbms/tests/queries/0_stateless/00965_pocopatch_logs_level_bugfix.sh similarity index 100% rename from dbms/tests/queries/0_stateless/00965_logs_level_bugfix.sh rename to dbms/tests/queries/0_stateless/00965_pocopatch_logs_level_bugfix.sh diff --git a/dbms/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.reference b/dbms/tests/queries/0_stateless/00965_pocopatch_send_logs_level_concurrent_queries.reference similarity index 100% rename from dbms/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.reference rename to dbms/tests/queries/0_stateless/00965_pocopatch_send_logs_level_concurrent_queries.reference diff --git a/dbms/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.sh b/dbms/tests/queries/0_stateless/00965_pocopatch_send_logs_level_concurrent_queries.sh similarity index 100% rename from dbms/tests/queries/0_stateless/00965_send_logs_level_concurrent_queries.sh rename to dbms/tests/queries/0_stateless/00965_pocopatch_send_logs_level_concurrent_queries.sh diff --git a/libs/libcommon/include/common/logger_useful.h b/libs/libcommon/include/common/logger_useful.h index 0878c589397..88a65563b2a 100644 --- a/libs/libcommon/include/common/logger_useful.h +++ b/libs/libcommon/include/common/logger_useful.h @@ -20,6 +20,40 @@ using DB::CurrentThread; /// Logs a message to a specified logger with that level. +#if defined(POCO_CLICKHOUSE_PATCH) + +#define LOG_TRACE(logger, message) do { \ + if ((logger)->trace()) {\ + std::stringstream oss_internal_rare; \ + oss_internal_rare << message; \ + (logger)->trace(oss_internal_rare.str());}} while(false) + +#define LOG_DEBUG(logger, message) do { \ + if ((logger)->debug()) {\ + std::stringstream oss_internal_rare; \ + oss_internal_rare << message; \ + (logger)->debug(oss_internal_rare.str());}} while(false) + +#define LOG_INFO(logger, message) do { \ + if ((logger)->information()) {\ + std::stringstream oss_internal_rare; \ + oss_internal_rare << message; \ + (logger)->information(oss_internal_rare.str());}} while(false) + +#define LOG_WARNING(logger, message) do { \ + if ((logger)->warning()) {\ + std::stringstream oss_internal_rare; \ + oss_internal_rare << message; \ + (logger)->warning(oss_internal_rare.str());}} while(false) + +#define LOG_ERROR(logger, message) do { \ + if ((logger)->error()) {\ + std::stringstream oss_internal_rare; \ + oss_internal_rare << message; \ + (logger)->error(oss_internal_rare.str());}} while(false) + +#else + #define LOG_TRACE(logger, message) do { \ const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && (CurrentThread::getGroup()->client_logs_level >= LogsLevel::trace); \ if ((logger)->trace() || is_clients_log) {\ @@ -74,3 +108,6 @@ using DB::CurrentThread; } else { \ (logger)->error(oss_internal_rare.str()); \ }}} while(false) + +#endif + From bd821d189873987071a52eae122f3bcd3b0545d4 Mon Sep 17 00:00:00 2001 From: chertus Date: Mon, 15 Jul 2019 21:57:00 +0300 Subject: [PATCH 18/64] exception safety and fix sanitizers' builds --- dbms/src/Common/new_delete.cpp | 59 +++++++++++++++++--------- libs/libcommon/include/common/memory.h | 23 +++++----- 2 files changed, 51 insertions(+), 31 deletions(-) diff --git a/dbms/src/Common/new_delete.cpp b/dbms/src/Common/new_delete.cpp index f790eef85cf..6a5a881cbf4 100644 --- a/dbms/src/Common/new_delete.cpp +++ b/dbms/src/Common/new_delete.cpp @@ -11,34 +11,47 @@ namespace Memory { -ALWAYS_INLINE void * trackMemory(void * ptr [[maybe_unused]], std::size_t size [[maybe_unused]]) noexcept +ALWAYS_INLINE void trackMemory(std::size_t size) { -#ifdef USE_JEMALLOC - if (likely(ptr != nullptr)) - CurrentMemoryTracker::alloc(sallocx(ptr, 0)); +#if USE_JEMALLOC + /// The nallocx() function allocates no memory, but it performs the same size computation as the mallocx() function + /// @note je_mallocx() != je_malloc(). It's expected they don't differ much in allocation logic. + CurrentMemoryTracker::alloc(nallocx(size, 0)); #else CurrentMemoryTracker::alloc(size); #endif - - return ptr; } -ALWAYS_INLINE void untrackMemory(void * ptr [[maybe_unused]]) noexcept +ALWAYS_INLINE bool trackMemoryNoExept(std::size_t size) noexcept { -#ifdef USE_JEMALLOC - if (likely(ptr != nullptr)) - CurrentMemoryTracker::free(sallocx(ptr, 0)); -#endif + try + { + if (likely(size != 0)) + trackMemory(size); + } + catch (...) + { + return false; + } + + return true; } -ALWAYS_INLINE void untrackMemory(void * ptr [[maybe_unused]], std::size_t size [[maybe_unused]]) noexcept +ALWAYS_INLINE void untrackMemory(void * ptr [[maybe_unused]], std::size_t size [[maybe_unused]] = 0) noexcept { -#ifdef USE_JEMALLOC - if (likely(ptr != nullptr)) - CurrentMemoryTracker::free(sallocx(ptr, 0)); + try + { +#if USE_JEMALLOC + /// @note It's also possible to use je_malloc_usable_size() here. + if (likely(ptr != nullptr)) + CurrentMemoryTracker::free(sallocx(ptr, 0)); #else - CurrentMemoryTracker::free(size); + if (size) + CurrentMemoryTracker::free(size); #endif + } + catch (...) + {} } } @@ -47,22 +60,28 @@ ALWAYS_INLINE void untrackMemory(void * ptr [[maybe_unused]], std::size_t size [ void * operator new(std::size_t size) { - return Memory::trackMemory(Memory::newImpl(size), size); + Memory::trackMemory(size); + return Memory::newImpl(size); } void * operator new[](std::size_t size) { - return Memory::trackMemory(Memory::newImpl(size), size); + Memory::trackMemory(size); + return Memory::newImpl(size); } void * operator new(std::size_t size, const std::nothrow_t &) noexcept { - return Memory::trackMemory(Memory::newNoExept(size), size); + if (likely(Memory::trackMemoryNoExept(size))) + return Memory::newNoExept(size); + return nullptr; } void * operator new[](std::size_t size, const std::nothrow_t &) noexcept { - return Memory::trackMemory(Memory::newNoExept(size), size); + if (likely(Memory::trackMemoryNoExept(size))) + return Memory::newNoExept(size); + return nullptr; } /// delete diff --git a/libs/libcommon/include/common/memory.h b/libs/libcommon/include/common/memory.h index bd4c3ac578c..bcb49b957b2 100644 --- a/libs/libcommon/include/common/memory.h +++ b/libs/libcommon/include/common/memory.h @@ -1,15 +1,12 @@ #pragma once +#include +#include + #if __has_include() #include #endif -#if USE_JEMALLOC - -#include -#include -#include - #if defined(_MSC_VER) #define ALWAYS_INLINE __forceinline #define NO_INLINE static __declspec(noinline) @@ -18,6 +15,10 @@ #define NO_INLINE __attribute__((__noinline__)) #endif +#if USE_JEMALLOC + +#include + namespace JeMalloc { @@ -25,7 +26,7 @@ void * handleOOM(std::size_t size, bool nothrow); ALWAYS_INLINE void * newImpl(std::size_t size) { - void * ptr = je_malloc(size); + void * ptr = malloc(size); if (likely(ptr != nullptr)) return ptr; @@ -34,7 +35,7 @@ ALWAYS_INLINE void * newImpl(std::size_t size) ALWAYS_INLINE void * newNoExept(std::size_t size) noexcept { - void * ptr = je_malloc(size); + void * ptr = malloc(size); if (likely(ptr != nullptr)) return ptr; @@ -43,7 +44,7 @@ ALWAYS_INLINE void * newNoExept(std::size_t size) noexcept ALWAYS_INLINE void deleteImpl(void * ptr) noexcept { - je_free(ptr); + free(ptr); } ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size) noexcept @@ -51,7 +52,7 @@ ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size) noexcept if (unlikely(ptr == nullptr)) return; - je_sdallocx(ptr, size, 0); + sdallocx(ptr, size, 0); } } @@ -86,7 +87,7 @@ ALWAYS_INLINE void deleteImpl(void * ptr) noexcept free(ptr); } -ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size) noexcept +ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size [[maybe_unused]]) noexcept { free(ptr); } From 69356759178c5abb444d76d0f86165743b08905e Mon Sep 17 00:00:00 2001 From: Nikita Mikhailov Date: Tue, 16 Jul 2019 11:36:50 +0300 Subject: [PATCH 19/64] small commit to fix build check --- libs/libcommon/include/common/logger_useful.h | 65 ++++++++++--------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/libs/libcommon/include/common/logger_useful.h b/libs/libcommon/include/common/logger_useful.h index 88a65563b2a..84c672b2c6c 100644 --- a/libs/libcommon/include/common/logger_useful.h +++ b/libs/libcommon/include/common/logger_useful.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -22,38 +23,6 @@ using DB::CurrentThread; #if defined(POCO_CLICKHOUSE_PATCH) -#define LOG_TRACE(logger, message) do { \ - if ((logger)->trace()) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - (logger)->trace(oss_internal_rare.str());}} while(false) - -#define LOG_DEBUG(logger, message) do { \ - if ((logger)->debug()) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - (logger)->debug(oss_internal_rare.str());}} while(false) - -#define LOG_INFO(logger, message) do { \ - if ((logger)->information()) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - (logger)->information(oss_internal_rare.str());}} while(false) - -#define LOG_WARNING(logger, message) do { \ - if ((logger)->warning()) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - (logger)->warning(oss_internal_rare.str());}} while(false) - -#define LOG_ERROR(logger, message) do { \ - if ((logger)->error()) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - (logger)->error(oss_internal_rare.str());}} while(false) - -#else - #define LOG_TRACE(logger, message) do { \ const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && (CurrentThread::getGroup()->client_logs_level >= LogsLevel::trace); \ if ((logger)->trace() || is_clients_log) {\ @@ -109,5 +78,37 @@ using DB::CurrentThread; (logger)->error(oss_internal_rare.str()); \ }}} while(false) +#else + +#define LOG_TRACE(logger, message) do { \ + if ((logger)->trace()) {\ + std::stringstream oss_internal_rare; \ + oss_internal_rare << message; \ + (logger)->trace(oss_internal_rare.str());}} while(false) + +#define LOG_DEBUG(logger, message) do { \ + if ((logger)->debug()) {\ + std::stringstream oss_internal_rare; \ + oss_internal_rare << message; \ + (logger)->debug(oss_internal_rare.str());}} while(false) + +#define LOG_INFO(logger, message) do { \ + if ((logger)->information()) {\ + std::stringstream oss_internal_rare; \ + oss_internal_rare << message; \ + (logger)->information(oss_internal_rare.str());}} while(false) + +#define LOG_WARNING(logger, message) do { \ + if ((logger)->warning()) {\ + std::stringstream oss_internal_rare; \ + oss_internal_rare << message; \ + (logger)->warning(oss_internal_rare.str());}} while(false) + +#define LOG_ERROR(logger, message) do { \ + if ((logger)->error()) {\ + std::stringstream oss_internal_rare; \ + oss_internal_rare << message; \ + (logger)->error(oss_internal_rare.str());}} while(false) + #endif From 7065504d5ea75c8f55291a3e6c176fa297d5f582 Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 16 Jul 2019 14:56:46 +0300 Subject: [PATCH 20/64] fix throw from ThreadStatus dtor --- dbms/src/Common/ThreadStatus.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/dbms/src/Common/ThreadStatus.cpp b/dbms/src/Common/ThreadStatus.cpp index 2e57a5237b4..e9408d09bd5 100644 --- a/dbms/src/Common/ThreadStatus.cpp +++ b/dbms/src/Common/ThreadStatus.cpp @@ -50,10 +50,18 @@ ThreadStatus::ThreadStatus() ThreadStatus::~ThreadStatus() { - if (untracked_memory > 0) - memory_tracker.alloc(untracked_memory); - else - memory_tracker.free(-untracked_memory); + try + { + if (untracked_memory > 0) + memory_tracker.alloc(untracked_memory); + else + memory_tracker.free(-untracked_memory); + } + catch (const DB::Exception &) + { + /// It's a minor tracked memory leak here (not the memory itself but it's counter). + /// We've already allocated a little bit more then the limit and cannot track it in the thread memory tracker or its parent. + } if (deleter) deleter(); From 4bc79bca35a046ed2e657e6edf3f332fe60873e9 Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 16 Jul 2019 17:18:01 +0300 Subject: [PATCH 21/64] fix unbandled build & add comment --- dbms/src/Common/MemoryTracker.cpp | 1 + dbms/src/Common/config.h.in | 1 - libs/libcommon/include/common/memory.h | 12 ++++++++++-- libs/libcommon/src/jemalloc_cpp.cpp | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/dbms/src/Common/MemoryTracker.cpp b/dbms/src/Common/MemoryTracker.cpp index 380dee75748..f59e4789b43 100644 --- a/dbms/src/Common/MemoryTracker.cpp +++ b/dbms/src/Common/MemoryTracker.cpp @@ -17,6 +17,7 @@ namespace DB static constexpr size_t log_peak_memory_usage_every = 1ULL << 30; +/// Each thread could new/delete memory in range of (-untracked_memory_limit, untracked_memory_limit) without access to common counters. static constexpr Int64 untracked_memory_limit = 4 * 1024 * 1024; diff --git a/dbms/src/Common/config.h.in b/dbms/src/Common/config.h.in index 556eeebec58..9b38dd9fc04 100644 --- a/dbms/src/Common/config.h.in +++ b/dbms/src/Common/config.h.in @@ -9,6 +9,5 @@ #cmakedefine01 USE_CPUINFO #cmakedefine01 USE_BROTLI #cmakedefine01 USE_MIMALLOC -#cmakedefine01 USE_JEMALLOC #cmakedefine01 CLICKHOUSE_SPLIT_BINARY diff --git a/libs/libcommon/include/common/memory.h b/libs/libcommon/include/common/memory.h index bcb49b957b2..93a8c600754 100644 --- a/libs/libcommon/include/common/memory.h +++ b/libs/libcommon/include/common/memory.h @@ -7,6 +7,16 @@ #include #endif +#if USE_JEMALLOC +#include + +#if JEMALLOC_VERSION_MAJOR < 4 + #undef USE_JEMALLOC + #define USE_JEMALLOC 0 + #include +#endif +#endif + #if defined(_MSC_VER) #define ALWAYS_INLINE __forceinline #define NO_INLINE static __declspec(noinline) @@ -17,8 +27,6 @@ #if USE_JEMALLOC -#include - namespace JeMalloc { diff --git a/libs/libcommon/src/jemalloc_cpp.cpp b/libs/libcommon/src/jemalloc_cpp.cpp index fbf074a0fb5..fdf941652c7 100644 --- a/libs/libcommon/src/jemalloc_cpp.cpp +++ b/libs/libcommon/src/jemalloc_cpp.cpp @@ -5,7 +5,7 @@ namespace JeMalloc { -void * handleOOM(std::size_t size, bool nothrow) +NO_INLINE void * handleOOM(std::size_t size, bool nothrow) { void * ptr = nullptr; From 04de34a74fe619b15999689de1df93cb1f346fb4 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 16 Jul 2019 19:27:42 +0300 Subject: [PATCH 22/64] changes after review --- contrib/poco | 2 +- dbms/programs/server/TCPHandler.cpp | 2 +- dbms/src/Common/ThreadStatus.h | 2 +- .../00965_pocopatch_logs_level_bugfix.sh | 2 +- ...atch_send_logs_level_concurrent_queries.sh | 4 +- libs/libcommon/include/common/logger_useful.h | 103 +++--------------- 6 files changed, 20 insertions(+), 95 deletions(-) diff --git a/contrib/poco b/contrib/poco index 4ea95983f9a..ece721f1085 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 4ea95983f9a92c9e5a21b6e782b4b3de330230ca +Subproject commit ece721f1085e3894cb5286e8560af84cd1445326 diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index 203665c2e1c..837812f84af 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -57,7 +57,7 @@ void TCPHandler::runImpl() ThreadStatus thread_status; connection_context = server.context(); - connection_context.setSessionContext(connection_context); + connection_context.makeSessionContext(); Settings global_settings = connection_context.getSettings(); diff --git a/dbms/src/Common/ThreadStatus.h b/dbms/src/Common/ThreadStatus.h index dbbbfca4988..77c816b2b2b 100644 --- a/dbms/src/Common/ThreadStatus.h +++ b/dbms/src/Common/ThreadStatus.h @@ -64,7 +64,7 @@ public: UInt32 master_thread_number = 0; Int32 master_thread_os_id = -1; - std::atomic client_logs_level = LogsLevel::none; + LogsLevel client_logs_level = LogsLevel::none; String query; }; diff --git a/dbms/tests/queries/0_stateless/00965_pocopatch_logs_level_bugfix.sh b/dbms/tests/queries/0_stateless/00965_pocopatch_logs_level_bugfix.sh index 50d1df6ce80..99de8cfb2d5 100755 --- a/dbms/tests/queries/0_stateless/00965_pocopatch_logs_level_bugfix.sh +++ b/dbms/tests/queries/0_stateless/00965_pocopatch_logs_level_bugfix.sh @@ -3,7 +3,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -cp /dev/null 00965_logs_level_bugfix.tmp +> 00965_logs_level_bugfix.tmp clickhouse-client --send_logs_level="trace" --query="SELECT 1;" 2>> 00965_logs_level_bugfix.tmp clickhouse-client --send_logs_level="debug" --query="SELECT 1;" 2>> 00965_logs_level_bugfix.tmp diff --git a/dbms/tests/queries/0_stateless/00965_pocopatch_send_logs_level_concurrent_queries.sh b/dbms/tests/queries/0_stateless/00965_pocopatch_send_logs_level_concurrent_queries.sh index 97343c5acd7..48b6b41282c 100755 --- a/dbms/tests/queries/0_stateless/00965_pocopatch_send_logs_level_concurrent_queries.sh +++ b/dbms/tests/queries/0_stateless/00965_pocopatch_send_logs_level_concurrent_queries.sh @@ -3,8 +3,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -cp /dev/null 00965_send_logs_level_concurrent_queries_first.tmp -cp /dev/null 00965_send_logs_level_concurrent_queries_second.tmp +> 00965_send_logs_level_concurrent_queries_first.tmp +> 00965_send_logs_level_concurrent_queries_second.tmp clickhouse-client --send_logs_level="trace" --query="SELECT * from numbers(100000);" >> /dev/null 2>> 00965_send_logs_level_concurrent_queries_first.tmp & clickhouse-client --send_logs_level="information" --query="SELECT * from numbers(100000);" >> /dev/null 2>> 00965_send_logs_level_concurrent_queries_second.tmp diff --git a/libs/libcommon/include/common/logger_useful.h b/libs/libcommon/include/common/logger_useful.h index 84c672b2c6c..213e0264279 100644 --- a/libs/libcommon/include/common/logger_useful.h +++ b/libs/libcommon/include/common/logger_useful.h @@ -3,7 +3,6 @@ /// Macros for convenient usage of Poco logger. #include -#include #include #include #include @@ -21,94 +20,20 @@ using DB::CurrentThread; /// Logs a message to a specified logger with that level. -#if defined(POCO_CLICKHOUSE_PATCH) +#define LOG_SIMPLE(logger, message, priority, PRIORITY) \ + const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && \ + (CurrentThread::getGroup()->client_logs_level >= (priority)); \ + if ((logger)->is((PRIORITY)) || is_clients_log) { \ + std::stringstream oss_internal_rare; \ + oss_internal_rare << message; \ + auto channel = (logger)->getChannel(); \ + channel->log(Message("", oss_internal_rare.str(), (PRIORITY))); \ + } \ -#define LOG_TRACE(logger, message) do { \ - const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && (CurrentThread::getGroup()->client_logs_level >= LogsLevel::trace); \ - if ((logger)->trace() || is_clients_log) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - if (is_clients_log) {\ - (logger)->force_log(oss_internal_rare.str(), Message::PRIO_TRACE); \ - } else { \ - (logger)->trace(oss_internal_rare.str()); \ - }}} while(false) +#define LOG_WARNING(logger, message) do { LOG_SIMPLE(logger, message, LogsLevel::warning, Message::PRIO_WARNING ) } while(false) +#define LOG_TRACE(logger, message) do { LOG_SIMPLE(logger, message, LogsLevel::trace, Message::PRIO_TRACE) } while(false) +#define LOG_DEBUG(logger, message) do { LOG_SIMPLE(logger, message, LogsLevel::debug, Message::PRIO_DEBUG) } while(false) +#define LOG_INFO(logger, message) do { LOG_SIMPLE(logger, message, LogsLevel::information, Message::PRIO_INFORMATION) } while(false) +#define LOG_ERROR(logger, message) do { LOG_SIMPLE(logger, message, LogsLevel::error, Message::PRIO_ERROR) } while(false) -#define LOG_DEBUG(logger, message) do { \ - const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && (CurrentThread::getGroup()->client_logs_level >= LogsLevel::debug); \ - if ((logger)->debug() || is_clients_log) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - if (is_clients_log) {\ - (logger)->force_log(oss_internal_rare.str(), Message::PRIO_DEBUG); \ - } else { \ - (logger)->debug(oss_internal_rare.str()); \ - }}} while(false) - -#define LOG_INFO(logger, message) do { \ - const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && (CurrentThread::getGroup()->client_logs_level >= LogsLevel::information); \ - if ((logger)->information() || is_clients_log) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - if (is_clients_log) {\ - (logger)->force_log(oss_internal_rare.str(), Message::PRIO_INFORMATION); \ - } else { \ - (logger)->information(oss_internal_rare.str()); \ - }}} while(false) - -#define LOG_WARNING(logger, message) do { \ - const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && (CurrentThread::getGroup()->client_logs_level >= LogsLevel::warning); \ - if ((logger)->warning() || is_clients_log) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - if (is_clients_log) {\ - (logger)->force_log(oss_internal_rare.str(), Message::PRIO_WARNING); \ - } else { \ - (logger)->warning(oss_internal_rare.str()); \ - }}} while(false) - -#define LOG_ERROR(logger, message) do { \ - const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && (CurrentThread::getGroup()->client_logs_level >= LogsLevel::error); \ - if ((logger)->error() || is_clients_log) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - if (is_clients_log) {\ - (logger)->force_log(oss_internal_rare.str(), Message::PRIO_ERROR); \ - } else { \ - (logger)->error(oss_internal_rare.str()); \ - }}} while(false) - -#else - -#define LOG_TRACE(logger, message) do { \ - if ((logger)->trace()) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - (logger)->trace(oss_internal_rare.str());}} while(false) - -#define LOG_DEBUG(logger, message) do { \ - if ((logger)->debug()) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - (logger)->debug(oss_internal_rare.str());}} while(false) - -#define LOG_INFO(logger, message) do { \ - if ((logger)->information()) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - (logger)->information(oss_internal_rare.str());}} while(false) - -#define LOG_WARNING(logger, message) do { \ - if ((logger)->warning()) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - (logger)->warning(oss_internal_rare.str());}} while(false) - -#define LOG_ERROR(logger, message) do { \ - if ((logger)->error()) {\ - std::stringstream oss_internal_rare; \ - oss_internal_rare << message; \ - (logger)->error(oss_internal_rare.str());}} while(false) - -#endif From 019c156afacbc50478a933ff8bb40f67f92e2989 Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 16 Jul 2019 19:36:10 +0300 Subject: [PATCH 23/64] fix je_nalocx() call with 0 & remove std::new_handler logic --- dbms/src/Common/new_delete.cpp | 6 +-- libs/libcommon/CMakeLists.txt | 1 - libs/libcommon/include/common/memory.h | 72 ++++++-------------------- libs/libcommon/src/jemalloc_cpp.cpp | 37 ------------- 4 files changed, 19 insertions(+), 97 deletions(-) delete mode 100644 libs/libcommon/src/jemalloc_cpp.cpp diff --git a/dbms/src/Common/new_delete.cpp b/dbms/src/Common/new_delete.cpp index 6a5a881cbf4..7cab8bb5b0c 100644 --- a/dbms/src/Common/new_delete.cpp +++ b/dbms/src/Common/new_delete.cpp @@ -16,7 +16,8 @@ ALWAYS_INLINE void trackMemory(std::size_t size) #if USE_JEMALLOC /// The nallocx() function allocates no memory, but it performs the same size computation as the mallocx() function /// @note je_mallocx() != je_malloc(). It's expected they don't differ much in allocation logic. - CurrentMemoryTracker::alloc(nallocx(size, 0)); + if (likely(size != 0)) + CurrentMemoryTracker::alloc(nallocx(size, 0)); #else CurrentMemoryTracker::alloc(size); #endif @@ -26,8 +27,7 @@ ALWAYS_INLINE bool trackMemoryNoExept(std::size_t size) noexcept { try { - if (likely(size != 0)) - trackMemory(size); + trackMemory(size); } catch (...) { diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index e388b30db1c..1e3be7f61d6 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -23,7 +23,6 @@ add_library(common src/getThreadNumber.cpp src/argsToConfig.cpp src/StackTrace.cpp - src/jemalloc_cpp.cpp include/common/SimpleCache.h include/common/StackTrace.h diff --git a/libs/libcommon/include/common/memory.h b/libs/libcommon/include/common/memory.h index 93a8c600754..d8dced79cfb 100644 --- a/libs/libcommon/include/common/memory.h +++ b/libs/libcommon/include/common/memory.h @@ -17,60 +17,8 @@ #endif #endif -#if defined(_MSC_VER) - #define ALWAYS_INLINE __forceinline - #define NO_INLINE static __declspec(noinline) -#else - #define ALWAYS_INLINE inline __attribute__((__always_inline__)) - #define NO_INLINE __attribute__((__noinline__)) -#endif - -#if USE_JEMALLOC - -namespace JeMalloc -{ - -void * handleOOM(std::size_t size, bool nothrow); - -ALWAYS_INLINE void * newImpl(std::size_t size) -{ - void * ptr = malloc(size); - if (likely(ptr != nullptr)) - return ptr; - - return handleOOM(size, false); -} - -ALWAYS_INLINE void * newNoExept(std::size_t size) noexcept -{ - void * ptr = malloc(size); - if (likely(ptr != nullptr)) - return ptr; - - return handleOOM(size, true); -} - -ALWAYS_INLINE void deleteImpl(void * ptr) noexcept -{ - free(ptr); -} - -ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size) noexcept -{ - if (unlikely(ptr == nullptr)) - return; - - sdallocx(ptr, size, 0); -} - -} - -namespace Memory -{ - using namespace JeMalloc; -} - -#else +#define ALWAYS_INLINE inline __attribute__((__always_inline__)) +#define NO_INLINE __attribute__((__noinline__)) namespace Memory { @@ -95,11 +43,23 @@ ALWAYS_INLINE void deleteImpl(void * ptr) noexcept free(ptr); } +#if USE_JEMALLOC + +ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size) noexcept +{ + if (unlikely(ptr == nullptr)) + return; + + sdallocx(ptr, size, 0); +} + +#else + ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size [[maybe_unused]]) noexcept { free(ptr); } -} - #endif + +} diff --git a/libs/libcommon/src/jemalloc_cpp.cpp b/libs/libcommon/src/jemalloc_cpp.cpp deleted file mode 100644 index fdf941652c7..00000000000 --- a/libs/libcommon/src/jemalloc_cpp.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include - -#if USE_JEMALLOC - -namespace JeMalloc -{ - -NO_INLINE void * handleOOM(std::size_t size, bool nothrow) -{ - void * ptr = nullptr; - - while (ptr == nullptr) - { - std::new_handler handler = std::get_new_handler(); - if (handler == nullptr) - break; - - try - { - handler(); - } - catch (const std::bad_alloc &) - { - break; - } - - ptr = je_malloc(size); - } - - if (ptr == nullptr && !nothrow) - throw std::bad_alloc{}; - return ptr; -} - -} - -#endif // USE_JEMALLOC From ded18495592facc7d3b73eff8e3438c448146019 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 16 Jul 2019 19:45:24 +0300 Subject: [PATCH 24/64] better --- libs/libcommon/include/common/logger_useful.h | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/libs/libcommon/include/common/logger_useful.h b/libs/libcommon/include/common/logger_useful.h index 213e0264279..e06916f1d0e 100644 --- a/libs/libcommon/include/common/logger_useful.h +++ b/libs/libcommon/include/common/logger_useful.h @@ -20,20 +20,23 @@ using DB::CurrentThread; /// Logs a message to a specified logger with that level. -#define LOG_SIMPLE(logger, message, priority, PRIORITY) \ +#define LOG_SIMPLE(logger, message, priority, PRIORITY) do \ +{ \ const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && \ (CurrentThread::getGroup()->client_logs_level >= (priority)); \ if ((logger)->is((PRIORITY)) || is_clients_log) { \ std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ - auto channel = (logger)->getChannel(); \ - channel->log(Message("", oss_internal_rare.str(), (PRIORITY))); \ + if ( auto channel = (logger)->getChannel() ) { \ + channel->log(Message("", oss_internal_rare.str(), (PRIORITY))); \ + } \ } \ - -#define LOG_WARNING(logger, message) do { LOG_SIMPLE(logger, message, LogsLevel::warning, Message::PRIO_WARNING ) } while(false) -#define LOG_TRACE(logger, message) do { LOG_SIMPLE(logger, message, LogsLevel::trace, Message::PRIO_TRACE) } while(false) -#define LOG_DEBUG(logger, message) do { LOG_SIMPLE(logger, message, LogsLevel::debug, Message::PRIO_DEBUG) } while(false) -#define LOG_INFO(logger, message) do { LOG_SIMPLE(logger, message, LogsLevel::information, Message::PRIO_INFORMATION) } while(false) -#define LOG_ERROR(logger, message) do { LOG_SIMPLE(logger, message, LogsLevel::error, Message::PRIO_ERROR) } while(false) +} while (false) +#define LOG_TRACE(logger, message) LOG_SIMPLE(logger, message, LogsLevel::trace, Message::PRIO_TRACE) +#define LOG_DEBUG(logger, message) LOG_SIMPLE(logger, message, LogsLevel::debug, Message::PRIO_DEBUG) +#define LOG_INFO(logger, message) LOG_SIMPLE(logger, message, LogsLevel::information, Message::PRIO_INFORMATION) +#define LOG_WARNING(logger, message) LOG_SIMPLE(logger, message, LogsLevel::warning, Message::PRIO_WARNING) +#define LOG_ERROR(logger, message) LOG_SIMPLE(logger, message, LogsLevel::error, Message::PRIO_ERROR) + From b74dacf9c4055ea2c0460dde4627aa623a09a5ee Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 16 Jul 2019 20:47:18 +0300 Subject: [PATCH 25/64] Check dictionaries for modification with unlocked ExternalLoader::mutex --- dbms/src/Interpreters/ExternalLoader.cpp | 48 ++++++++++++++++++++---- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index c34193f71e4..612bd2362c8 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -528,16 +528,48 @@ public: /// The function doesn't touch the objects which were never tried to load. void reloadOutdated() { - std::lock_guard lock{mutex}; - TimePoint now = std::chrono::system_clock::now(); - for (auto & [name, info] : infos) - if ((now >= info.next_update_time) && !info.loading() && info.was_loading()) + std::unordered_map is_modified_map; + + { + std::lock_guard lock{mutex}; + TimePoint now = std::chrono::system_clock::now(); + for (const auto & name_and_info : infos) { - if (info.loaded() && !is_object_modified(info.object)) - info.next_update_time = calculate_next_update_time(info.object, info.error_count); - else - startLoading(name, info); + const auto & info = name_and_info.second; + if ((now >= info.next_update_time) && !info.loading() && info.was_loading()) + is_modified_map.emplace(info.object, true); } + } + + /// The `mutex` should be unlocked while we're calling the function is_object_modified(). + for (auto & [object, is_modified_flag] : is_modified_map) + { + try + { + is_modified_flag = is_object_modified(object); + } + catch (...) + { + tryLogCurrentException(log, "Could not check if " + type_name + " '" + object->getName() + "' was modified"); + } + } + + { + std::lock_guard lock{mutex}; + TimePoint now = std::chrono::system_clock::now(); + for (auto & [name, info] : infos) + if ((now >= info.next_update_time) && !info.loading() && info.was_loading()) + { + auto it = is_modified_map.find(info.object); + if (it == is_modified_map.end()) + continue; /// Object has been just added, it can be simply omitted from this update of outdated. + bool is_modified_flag = it->second; + if (info.loaded() && !is_modified_flag) + info.next_update_time = calculate_next_update_time(info.object, info.error_count); + else + startLoading(name, info); + } + } } private: From 96f7a41044547801febc00719d38417206ab0ad3 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 16 Jul 2019 20:57:08 +0300 Subject: [PATCH 26/64] Add an integration test for the invalidate_query parameter. --- .../dictionaries/dictionary_preset_dep_x.xml | 4 +- .../dictionaries/dictionary_preset_dep_y.xml | 18 ++++---- .../dictionaries/dictionary_preset_dep_z.xml | 11 ++--- .../integration/test_dictionaries/test.py | 41 +++++++++++++++---- 4 files changed, 51 insertions(+), 23 deletions(-) diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_x.xml b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_x.xml index 36a421f1908..6eed3fbc891 100644 --- a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_x.xml +++ b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_x.xml @@ -12,7 +12,7 @@ dep_z
- 60 + 5 @@ -21,7 +21,7 @@ id - String_ + a String XX diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_y.xml b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_y.xml index 771ab889660..7891e945566 100644 --- a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_y.xml +++ b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_y.xml @@ -9,10 +9,10 @@ default test - dictionary_source
+ small_dict_source
- 60 + 5 @@ -21,17 +21,17 @@ id - Int64_ - Int64 - 121 + b + Int32 + -1 - Float32_ - Float32 - 121 + c + Float64 + -2 - String_ + a String YY diff --git a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_z.xml b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_z.xml index 875d55c39f9..e17107d2ac2 100644 --- a/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_z.xml +++ b/dbms/tests/integration/test_dictionaries/configs/dictionaries/dictionary_preset_dep_z.xml @@ -10,9 +10,10 @@ dict dep_y
+ SELECT intDiv(count(), 5) from dict.dep_y - 60 + 5 @@ -21,12 +22,12 @@ id - Int64_ - Int64 - 122 + b + Int32 + -3 - String_ + a String ZZ diff --git a/dbms/tests/integration/test_dictionaries/test.py b/dbms/tests/integration/test_dictionaries/test.py index bf698bae452..dfb27cd2ed7 100644 --- a/dbms/tests/integration/test_dictionaries/test.py +++ b/dbms/tests/integration/test_dictionaries/test.py @@ -3,7 +3,7 @@ import os import time from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV +from helpers.test_tools import TSV, assert_eq_with_retry from generate_dictionaries import generate_structure, generate_dictionaries, DictionaryTestTable SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -54,6 +54,13 @@ def started_cluster(): for line in TSV(instance.query('select name from system.dictionaries')).lines: print line, + # Create table `test.small_dict_source` + instance.query(''' + drop table if exists test.small_dict_source; + create table test.small_dict_source (id UInt64, a String, b Int32, c Float64) engine=Log; + insert into test.small_dict_source values (0, 'water', 10, 1), (1, 'air', 40, 0.01), (2, 'earth', 100, 1.7); + ''') + yield cluster finally: @@ -166,17 +173,37 @@ def test_dictionary_dependency(started_cluster): # Dictionary 'dep_x' depends on 'dep_z', which depends on 'dep_y'. # So they all should be loaded at once. - assert query("SELECT dictGetString('dep_x', 'String_', toUInt64(1))") == "10577349846663553072\n" + assert query("SELECT dictGetString('dep_x', 'a', toUInt64(1))") == "air\n" assert get_status('dep_x') == 'LOADED' assert get_status('dep_y') == 'LOADED' assert get_status('dep_z') == 'LOADED' # Other dictionaries should work too. - assert query("SELECT dictGetString('dep_y', 'String_', toUInt64(1))") == "10577349846663553072\n" - assert query("SELECT dictGetString('dep_z', 'String_', toUInt64(1))") == "10577349846663553072\n" - assert query("SELECT dictGetString('dep_x', 'String_', toUInt64(12121212))") == "XX\n" - assert query("SELECT dictGetString('dep_y', 'String_', toUInt64(12121212))") == "YY\n" - assert query("SELECT dictGetString('dep_z', 'String_', toUInt64(12121212))") == "ZZ\n" + assert query("SELECT dictGetString('dep_y', 'a', toUInt64(1))") == "air\n" + assert query("SELECT dictGetString('dep_z', 'a', toUInt64(1))") == "air\n" + + assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "XX\n" + assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "YY\n" + assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "ZZ\n" + + # Update the source table. + query("insert into test.small_dict_source values (3, 'fire', 30, 8)") + + # Wait for dictionaries to be reloaded. + assert_eq_with_retry(instance, "SELECT dictHas('dep_y', toUInt64(3))", "1", sleep_time = 2, retry_count = 10) + assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "XX\n" + assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n" + assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "ZZ\n" + + # dep_x and dep_z are updated only when there `intDiv(count(), 4)` is changed. + query("insert into test.small_dict_source values (4, 'ether', 404, 0.001)") + assert_eq_with_retry(instance, "SELECT dictHas('dep_x', toUInt64(4))", "1", sleep_time = 2, retry_count = 10) + assert query("SELECT dictGetString('dep_x', 'a', toUInt64(3))") == "fire\n" + assert query("SELECT dictGetString('dep_y', 'a', toUInt64(3))") == "fire\n" + assert query("SELECT dictGetString('dep_z', 'a', toUInt64(3))") == "fire\n" + assert query("SELECT dictGetString('dep_x', 'a', toUInt64(4))") == "ether\n" + assert query("SELECT dictGetString('dep_y', 'a', toUInt64(4))") == "ether\n" + assert query("SELECT dictGetString('dep_z', 'a', toUInt64(4))") == "ether\n" def test_reload_while_loading(started_cluster): From 3db106c1f241cead73cd2296ec8d5d8d81d556aa Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 16 Jul 2019 21:09:06 +0300 Subject: [PATCH 27/64] disable new/delete memory tracking for unbundled build --- dbms/src/Common/new_delete.cpp | 2 +- libs/libcommon/include/common/config_common.h.in | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Common/new_delete.cpp b/dbms/src/Common/new_delete.cpp index 7cab8bb5b0c..d9140f9459d 100644 --- a/dbms/src/Common/new_delete.cpp +++ b/dbms/src/Common/new_delete.cpp @@ -6,7 +6,7 @@ /// Replace default new/delete with memory tracking versions. /// @sa https://en.cppreference.com/w/cpp/memory/new/operator_new /// https://en.cppreference.com/w/cpp/memory/new/operator_delete -#if 1 +#if NOT_UNBUNDLED namespace Memory { diff --git a/libs/libcommon/include/common/config_common.h.in b/libs/libcommon/include/common/config_common.h.in index 0cc0950efba..247afd87aea 100644 --- a/libs/libcommon/include/common/config_common.h.in +++ b/libs/libcommon/include/common/config_common.h.in @@ -7,3 +7,4 @@ #cmakedefine01 USE_READLINE #cmakedefine01 USE_LIBEDIT #cmakedefine01 HAVE_READLINE_HISTORY +#cmakedefine01 NOT_UNBUNDLED From 6d563c51e64b23002b16eef7620d0f0803785037 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 17 Jul 2019 11:39:36 +0300 Subject: [PATCH 28/64] Remove unnecessary try-catch in the constructors of classes derived from IExternalLoadable. --- dbms/src/Dictionaries/CacheDictionary.h | 6 ------ .../src/Dictionaries/ComplexKeyCacheDictionary.h | 4 ---- .../Dictionaries/ComplexKeyHashedDictionary.cpp | 14 ++------------ .../Dictionaries/ComplexKeyHashedDictionary.h | 8 -------- dbms/src/Dictionaries/FlatDictionary.cpp | 14 ++------------ dbms/src/Dictionaries/FlatDictionary.h | 8 -------- dbms/src/Dictionaries/HashedDictionary.cpp | 14 ++------------ dbms/src/Dictionaries/HashedDictionary.h | 8 -------- dbms/src/Dictionaries/RangeHashedDictionary.cpp | 14 ++------------ dbms/src/Dictionaries/RangeHashedDictionary.h | 8 -------- dbms/src/Dictionaries/TrieDictionary.h | 4 ---- dbms/src/Interpreters/CatBoostModel.cpp | 14 -------------- dbms/src/Interpreters/CatBoostModel.h | 6 ------ dbms/src/Interpreters/ExternalLoader.cpp | 16 +++++----------- dbms/src/Interpreters/ExternalLoader.h | 4 +--- dbms/src/Interpreters/IExternalLoadable.h | 5 ----- 16 files changed, 14 insertions(+), 133 deletions(-) diff --git a/dbms/src/Dictionaries/CacheDictionary.h b/dbms/src/Dictionaries/CacheDictionary.h index c6607196af5..cc613d0d96b 100644 --- a/dbms/src/Dictionaries/CacheDictionary.h +++ b/dbms/src/Dictionaries/CacheDictionary.h @@ -30,8 +30,6 @@ public: const DictionaryLifetime dict_lifetime, const size_t size); - std::exception_ptr getCreationException() const override { return {}; } - std::string getName() const override { return name; } std::string getTypeName() const override { return "Cache"; } @@ -62,8 +60,6 @@ public: const DictionaryStructure & getStructure() const override { return dict_struct; } - std::chrono::time_point getCreationTime() const override { return creation_time; } - bool isInjective(const std::string & attribute_name) const override { return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; @@ -284,8 +280,6 @@ private: mutable std::atomic element_count{0}; mutable std::atomic hit_count{0}; mutable std::atomic query_count{0}; - - const std::chrono::time_point creation_time = std::chrono::system_clock::now(); }; } diff --git a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h index 2c080d68b7b..ffac807c04c 100644 --- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h +++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h @@ -50,8 +50,6 @@ public: std::string getKeyDescription() const { return key_description; } - std::exception_ptr getCreationException() const override { return {}; } - std::string getName() const override { return name; } std::string getTypeName() const override { return "ComplexKeyCache"; } @@ -86,8 +84,6 @@ public: const DictionaryStructure & getStructure() const override { return dict_struct; } - std::chrono::time_point getCreationTime() const override { return creation_time; } - bool isInjective(const std::string & attribute_name) const override { return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; diff --git a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp index 606e10f2412..39ef9124061 100644 --- a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp +++ b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp @@ -29,18 +29,8 @@ ComplexKeyHashedDictionary::ComplexKeyHashedDictionary( , saved_block{std::move(saved_block)} { createAttributes(); - - try - { - loadData(); - calculateBytesAllocated(); - } - catch (...) - { - creation_exception = std::current_exception(); - } - - creation_time = std::chrono::system_clock::now(); + loadData(); + calculateBytesAllocated(); } #define DECLARE(TYPE) \ diff --git a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h index b9aaa42a829..54ee8627f9b 100644 --- a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h +++ b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h @@ -32,8 +32,6 @@ public: std::string getKeyDescription() const { return key_description; } - std::exception_ptr getCreationException() const override { return creation_exception; } - std::string getName() const override { return name; } std::string getTypeName() const override { return "ComplexKeyHashed"; } @@ -61,8 +59,6 @@ public: const DictionaryStructure & getStructure() const override { return dict_struct; } - std::chrono::time_point getCreationTime() const override { return creation_time; } - bool isInjective(const std::string & attribute_name) const override { return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; @@ -255,10 +251,6 @@ private: size_t bucket_count = 0; mutable std::atomic query_count{0}; - std::chrono::time_point creation_time; - - std::exception_ptr creation_exception; - BlockPtr saved_block; }; diff --git a/dbms/src/Dictionaries/FlatDictionary.cpp b/dbms/src/Dictionaries/FlatDictionary.cpp index 628178c8542..b7b70748c01 100644 --- a/dbms/src/Dictionaries/FlatDictionary.cpp +++ b/dbms/src/Dictionaries/FlatDictionary.cpp @@ -36,18 +36,8 @@ FlatDictionary::FlatDictionary( , saved_block{std::move(saved_block)} { createAttributes(); - - try - { - loadData(); - calculateBytesAllocated(); - } - catch (...) - { - creation_exception = std::current_exception(); - } - - creation_time = std::chrono::system_clock::now(); + loadData(); + calculateBytesAllocated(); } diff --git a/dbms/src/Dictionaries/FlatDictionary.h b/dbms/src/Dictionaries/FlatDictionary.h index 2a00de6f754..de14cc3dc1a 100644 --- a/dbms/src/Dictionaries/FlatDictionary.h +++ b/dbms/src/Dictionaries/FlatDictionary.h @@ -29,8 +29,6 @@ public: bool require_nonempty, BlockPtr saved_block = nullptr); - std::exception_ptr getCreationException() const override { return creation_exception; } - std::string getName() const override { return name; } std::string getTypeName() const override { return "Flat"; } @@ -58,8 +56,6 @@ public: const DictionaryStructure & getStructure() const override { return dict_struct; } - std::chrono::time_point getCreationTime() const override { return creation_time; } - bool isInjective(const std::string & attribute_name) const override { return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; @@ -244,10 +240,6 @@ private: size_t bucket_count = 0; mutable std::atomic query_count{0}; - std::chrono::time_point creation_time; - - std::exception_ptr creation_exception; - BlockPtr saved_block; }; diff --git a/dbms/src/Dictionaries/HashedDictionary.cpp b/dbms/src/Dictionaries/HashedDictionary.cpp index d67d6dcf9a2..413cfadec39 100644 --- a/dbms/src/Dictionaries/HashedDictionary.cpp +++ b/dbms/src/Dictionaries/HashedDictionary.cpp @@ -30,18 +30,8 @@ HashedDictionary::HashedDictionary( , saved_block{std::move(saved_block)} { createAttributes(); - - try - { - loadData(); - calculateBytesAllocated(); - } - catch (...) - { - creation_exception = std::current_exception(); - } - - creation_time = std::chrono::system_clock::now(); + loadData(); + calculateBytesAllocated(); } diff --git a/dbms/src/Dictionaries/HashedDictionary.h b/dbms/src/Dictionaries/HashedDictionary.h index b0605f26bad..92875f27cf3 100644 --- a/dbms/src/Dictionaries/HashedDictionary.h +++ b/dbms/src/Dictionaries/HashedDictionary.h @@ -28,8 +28,6 @@ public: bool require_nonempty, BlockPtr saved_block = nullptr); - std::exception_ptr getCreationException() const override { return creation_exception; } - std::string getName() const override { return name; } std::string getTypeName() const override { return "Hashed"; } @@ -57,8 +55,6 @@ public: const DictionaryStructure & getStructure() const override { return dict_struct; } - std::chrono::time_point getCreationTime() const override { return creation_time; } - bool isInjective(const std::string & attribute_name) const override { return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; @@ -248,10 +244,6 @@ private: size_t bucket_count = 0; mutable std::atomic query_count{0}; - std::chrono::time_point creation_time; - - std::exception_ptr creation_exception; - BlockPtr saved_block; }; diff --git a/dbms/src/Dictionaries/RangeHashedDictionary.cpp b/dbms/src/Dictionaries/RangeHashedDictionary.cpp index ac509b4d1e5..05f29e05c42 100644 --- a/dbms/src/Dictionaries/RangeHashedDictionary.cpp +++ b/dbms/src/Dictionaries/RangeHashedDictionary.cpp @@ -80,18 +80,8 @@ RangeHashedDictionary::RangeHashedDictionary( , require_nonempty(require_nonempty) { createAttributes(); - - try - { - loadData(); - calculateBytesAllocated(); - } - catch (...) - { - creation_exception = std::current_exception(); - } - - creation_time = std::chrono::system_clock::now(); + loadData(); + calculateBytesAllocated(); } diff --git a/dbms/src/Dictionaries/RangeHashedDictionary.h b/dbms/src/Dictionaries/RangeHashedDictionary.h index b54c88de4e8..a02b1377db5 100644 --- a/dbms/src/Dictionaries/RangeHashedDictionary.h +++ b/dbms/src/Dictionaries/RangeHashedDictionary.h @@ -24,8 +24,6 @@ public: const DictionaryLifetime dict_lifetime, bool require_nonempty); - std::exception_ptr getCreationException() const override { return creation_exception; } - std::string getName() const override { return dictionary_name; } std::string getTypeName() const override { return "RangeHashed"; } @@ -53,8 +51,6 @@ public: const DictionaryStructure & getStructure() const override { return dict_struct; } - std::chrono::time_point getCreationTime() const override { return creation_time; } - bool isInjective(const std::string & attribute_name) const override { return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; @@ -227,10 +223,6 @@ private: size_t element_count = 0; size_t bucket_count = 0; mutable std::atomic query_count{0}; - - std::chrono::time_point creation_time; - - std::exception_ptr creation_exception; }; } diff --git a/dbms/src/Dictionaries/TrieDictionary.h b/dbms/src/Dictionaries/TrieDictionary.h index f434ebbc77d..a873f7bdd16 100644 --- a/dbms/src/Dictionaries/TrieDictionary.h +++ b/dbms/src/Dictionaries/TrieDictionary.h @@ -33,8 +33,6 @@ public: std::string getKeyDescription() const { return key_description; } - std::exception_ptr getCreationException() const override { return creation_exception; } - std::string getName() const override { return name; } std::string getTypeName() const override { return "Trie"; } @@ -62,8 +60,6 @@ public: const DictionaryStructure & getStructure() const override { return dict_struct; } - std::chrono::time_point getCreationTime() const override { return creation_time; } - bool isInjective(const std::string & attribute_name) const override { return dict_struct.attributes[&getAttribute(attribute_name) - attributes.data()].injective; diff --git a/dbms/src/Interpreters/CatBoostModel.cpp b/dbms/src/Interpreters/CatBoostModel.cpp index 7f98e5131ae..3e6e66b5c3f 100644 --- a/dbms/src/Interpreters/CatBoostModel.cpp +++ b/dbms/src/Interpreters/CatBoostModel.cpp @@ -504,20 +504,6 @@ std::shared_ptr getCatBoostWrapperHolder(const std::string & CatBoostModel::CatBoostModel(std::string name_, std::string model_path_, std::string lib_path_, const ExternalLoadableLifetime & lifetime) : name(std::move(name_)), model_path(std::move(model_path_)), lib_path(std::move(lib_path_)), lifetime(lifetime) -{ - try - { - init(); - } - catch (...) - { - creation_exception = std::current_exception(); - } - - creation_time = std::chrono::system_clock::now(); -} - -void CatBoostModel::init() { api_provider = getCatBoostWrapperHolder(lib_path); api = &api_provider->getAPI(); diff --git a/dbms/src/Interpreters/CatBoostModel.h b/dbms/src/Interpreters/CatBoostModel.h index 6f613ad0f24..541dd111c82 100644 --- a/dbms/src/Interpreters/CatBoostModel.h +++ b/dbms/src/Interpreters/CatBoostModel.h @@ -68,9 +68,6 @@ public: std::shared_ptr clone() const override; - std::chrono::time_point getCreationTime() const override { return creation_time; } - std::exception_ptr getCreationException() const override { return creation_exception; } - private: std::string name; std::string model_path; @@ -85,9 +82,6 @@ private: size_t cat_features_count; size_t tree_count; - std::chrono::time_point creation_time; - std::exception_ptr creation_exception; - void init(); }; diff --git a/dbms/src/Interpreters/ExternalLoader.cpp b/dbms/src/Interpreters/ExternalLoader.cpp index 612bd2362c8..c7b3d202a28 100644 --- a/dbms/src/Interpreters/ExternalLoader.cpp +++ b/dbms/src/Interpreters/ExternalLoader.cpp @@ -219,7 +219,7 @@ class ExternalLoader::LoadingDispatcher : private boost::noncopyable { public: /// Called to load or reload an object. - using CreateObjectFunction = std::function; /// Called after loading/reloading an object to calculate the time of the next update. @@ -783,7 +783,7 @@ private: std::exception_ptr new_exception; try { - std::tie(new_object, new_exception) = create_object(name, config, config_changed, previous_version); + new_object = create_object(name, config, config_changed, previous_version); } catch (...) { @@ -792,8 +792,6 @@ private: if (!new_object && !new_exception) throw Exception("No object created and no exception raised for " + type_name, ErrorCodes::LOGICAL_ERROR); - if (new_object && new_exception) - new_object = nullptr; /// Calculate a new update time. TimePoint next_update_time; @@ -1152,17 +1150,13 @@ void ExternalLoader::reload(bool load_never_loading) loading_dispatcher->reload(load_never_loading); } -ExternalLoader::ObjectWithException ExternalLoader::createObject( +ExternalLoader::LoadablePtr ExternalLoader::createObject( const String & name, const ObjectConfig & config, bool config_changed, const LoadablePtr & previous_version) const { if (previous_version && !config_changed) - { - auto new_object = previous_version->clone(); - return {new_object, new_object->getCreationException()}; - } + return previous_version->clone(); - auto new_object = create(name, *config.config, config.key_in_config); - return {new_object, new_object->getCreationException()}; + return create(name, *config.config, config.key_in_config); } ExternalLoader::TimePoint ExternalLoader::calculateNextUpdateTime(const LoadablePtr & loaded_object, size_t error_count) const diff --git a/dbms/src/Interpreters/ExternalLoader.h b/dbms/src/Interpreters/ExternalLoader.h index da999bfe21a..4c94b8d69cd 100644 --- a/dbms/src/Interpreters/ExternalLoader.h +++ b/dbms/src/Interpreters/ExternalLoader.h @@ -186,10 +186,8 @@ protected: private: struct ObjectConfig; - using ObjectWithException = std::pair; - ObjectWithException - createObject(const String & name, const ObjectConfig & config, bool config_changed, const LoadablePtr & previous_version) const; + LoadablePtr createObject(const String & name, const ObjectConfig & config, bool config_changed, const LoadablePtr & previous_version) const; TimePoint calculateNextUpdateTime(const LoadablePtr & loaded_object, size_t error_count) const; class ConfigFilesReader; diff --git a/dbms/src/Interpreters/IExternalLoadable.h b/dbms/src/Interpreters/IExternalLoadable.h index 7b875f02060..f8725a67989 100644 --- a/dbms/src/Interpreters/IExternalLoadable.h +++ b/dbms/src/Interpreters/IExternalLoadable.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -41,10 +40,6 @@ public: virtual bool isModified() const = 0; /// Returns new object with the same configuration. Is used to update modified object when lifetime exceeded. virtual std::shared_ptr clone() const = 0; - - virtual std::chrono::time_point getCreationTime() const = 0; - - virtual std::exception_ptr getCreationException() const = 0; }; } From e5360413a6488a22e7d6fb397aeb37419f8378ec Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 17 Jul 2019 14:55:18 +0300 Subject: [PATCH 29/64] integration test added --- dbms/tests/integration/helpers/client.py | 17 ++++++++++++ dbms/tests/integration/helpers/cluster.py | 4 +++ .../integration/test_logs_level/__init__.py | 0 .../configs/config_information.xml | 26 +++++++++++++++++++ .../tests/integration/test_logs_level/test.py | 18 +++++++++++++ libs/libcommon/include/common/logger_useful.h | 4 +-- 6 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 dbms/tests/integration/test_logs_level/__init__.py create mode 100644 dbms/tests/integration/test_logs_level/configs/config_information.xml create mode 100644 dbms/tests/integration/test_logs_level/test.py diff --git a/dbms/tests/integration/helpers/client.py b/dbms/tests/integration/helpers/client.py index 9df53c40e67..fd59166b137 100644 --- a/dbms/tests/integration/helpers/client.py +++ b/dbms/tests/integration/helpers/client.py @@ -44,6 +44,9 @@ class Client: return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user).get_error() + def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None): + return self.get_query_request(sql, stdin=stdin, timeout=timeout, settings=settings, user=user).get_answer_and_error() + class QueryTimeoutExceedException(Exception): pass @@ -110,3 +113,17 @@ class CommandRequest: raise QueryRuntimeException('Client expected to be failed but succeeded! stdout: {}'.format(stdout)) return stderr + + + def get_answer_and_error(self): + self.process.wait() + self.stdout_file.seek(0) + self.stderr_file.seek(0) + + stdout = self.stdout_file.read() + stderr = self.stderr_file.read() + + if self.timer is not None and not self.process_finished_before_timeout and not self.ignore_error: + raise QueryTimeoutExceedException('Client timed out!') + + return (stdout, stderr) diff --git a/dbms/tests/integration/helpers/cluster.py b/dbms/tests/integration/helpers/cluster.py index fbc6591eaa5..bd3ecb9ae9c 100644 --- a/dbms/tests/integration/helpers/cluster.py +++ b/dbms/tests/integration/helpers/cluster.py @@ -527,6 +527,10 @@ class ClickHouseInstance: def query_and_get_error(self, sql, stdin=None, timeout=None, settings=None, user=None): return self.client.query_and_get_error(sql, stdin, timeout, settings, user) + # The same as query_and_get_error but ignores successful query. + def query_and_get_answer_with_error(self, sql, stdin=None, timeout=None, settings=None, user=None): + return self.client.query_and_get_answer_with_error(sql, stdin, timeout, settings, user) + # Connects to the instance via HTTP interface, sends a query and returns the answer def http_query(self, sql, data=None): return urllib.urlopen("http://"+self.ip_address+":8123/?query="+urllib.quote(sql,safe=''), data).read() diff --git a/dbms/tests/integration/test_logs_level/__init__.py b/dbms/tests/integration/test_logs_level/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_logs_level/configs/config_information.xml b/dbms/tests/integration/test_logs_level/configs/config_information.xml new file mode 100644 index 00000000000..b99f57402fd --- /dev/null +++ b/dbms/tests/integration/test_logs_level/configs/config_information.xml @@ -0,0 +1,26 @@ + + + + information + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + + true + none + + AcceptCertificateHandler + + + + + 500 + 5368709120 + users.xml + + /etc/clickhouse-server/config.d/*.xml + diff --git a/dbms/tests/integration/test_logs_level/test.py b/dbms/tests/integration/test_logs_level/test.py new file mode 100644 index 00000000000..517b003a779 --- /dev/null +++ b/dbms/tests/integration/test_logs_level/test.py @@ -0,0 +1,18 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node', main_configs=['configs/config_information.xml']) + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +def test_check_client_logs_level(start_cluster): + logs = node.query_and_get_answer_with_error("SELECT 1", settings={"send_logs_level": 'trace'})[1] + assert logs.count('Trace') != 0 \ No newline at end of file diff --git a/libs/libcommon/include/common/logger_useful.h b/libs/libcommon/include/common/logger_useful.h index e06916f1d0e..c1c39047540 100644 --- a/libs/libcommon/include/common/logger_useful.h +++ b/libs/libcommon/include/common/logger_useful.h @@ -27,8 +27,8 @@ using DB::CurrentThread; if ((logger)->is((PRIORITY)) || is_clients_log) { \ std::stringstream oss_internal_rare; \ oss_internal_rare << message; \ - if ( auto channel = (logger)->getChannel() ) { \ - channel->log(Message("", oss_internal_rare.str(), (PRIORITY))); \ + if (auto channel = (logger)->getChannel()) { \ + channel->log(Message((logger)->name(), oss_internal_rare.str(), (PRIORITY))); \ } \ } \ } while (false) From 56c9bf5accab4e80a530c6865f2d4be21f30b2f4 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 17 Jul 2019 14:57:05 +0300 Subject: [PATCH 30/64] pep8 --- dbms/tests/integration/test_logs_level/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/integration/test_logs_level/test.py b/dbms/tests/integration/test_logs_level/test.py index 517b003a779..302686b6fa0 100644 --- a/dbms/tests/integration/test_logs_level/test.py +++ b/dbms/tests/integration/test_logs_level/test.py @@ -15,4 +15,4 @@ def start_cluster(): def test_check_client_logs_level(start_cluster): logs = node.query_and_get_answer_with_error("SELECT 1", settings={"send_logs_level": 'trace'})[1] - assert logs.count('Trace') != 0 \ No newline at end of file + assert logs.count('Trace') != 0 From ce1bc54c0c1701d6c95d58ec76fc2b80a737102c Mon Sep 17 00:00:00 2001 From: chertus Date: Wed, 17 Jul 2019 15:40:05 +0300 Subject: [PATCH 31/64] infinite loop detection in MemoryTracker + shrink joins perf test into 1Gb memory usage --- dbms/src/Common/MemoryTracker.cpp | 10 +++++++++- dbms/src/Common/MemoryTracker.h | 2 ++ dbms/tests/performance/joins_in_memory.xml | 10 +++++----- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/dbms/src/Common/MemoryTracker.cpp b/dbms/src/Common/MemoryTracker.cpp index f59e4789b43..91309af1d6c 100644 --- a/dbms/src/Common/MemoryTracker.cpp +++ b/dbms/src/Common/MemoryTracker.cpp @@ -1,3 +1,5 @@ +#include + #include "MemoryTracker.h" #include #include @@ -115,7 +117,12 @@ void MemoryTracker::alloc(Int64 size) message << " exceeded: would use " << formatReadableSizeWithBinarySuffix(will_be) << " (attempt to allocate chunk of " << size << " bytes)" << ", maximum: " << formatReadableSizeWithBinarySuffix(current_limit); - +#ifndef NDEBUG + /// Prevent infinite loop in case of catching & new in cycle. + if (last_attempt_failed) + std::abort(); +#endif + ++last_attempt_failed; throw DB::Exception(message.str(), DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED); } @@ -130,6 +137,7 @@ void MemoryTracker::alloc(Int64 size) if (auto loaded_next = parent.load(std::memory_order_relaxed)) loaded_next->alloc(size); + last_attempt_failed = 0; } diff --git a/dbms/src/Common/MemoryTracker.h b/dbms/src/Common/MemoryTracker.h index 4ce0ac262fa..73f46a447c7 100644 --- a/dbms/src/Common/MemoryTracker.h +++ b/dbms/src/Common/MemoryTracker.h @@ -115,6 +115,8 @@ public: /// To be able to temporarily stop memory tracker DB::SimpleActionBlocker blocker; + + UInt32 last_attempt_failed = 0; }; diff --git a/dbms/tests/performance/joins_in_memory.xml b/dbms/tests/performance/joins_in_memory.xml index 23b009a6027..1da400c48f4 100644 --- a/dbms/tests/performance/joins_in_memory.xml +++ b/dbms/tests/performance/joins_in_memory.xml @@ -13,11 +13,11 @@ CREATE TABLE ints (i64 Int64, i32 Int32, i16 Int16, i8 Int8) ENGINE = Memory - INSERT INTO ints SELECT number AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000) - INSERT INTO ints SELECT 10000 + number % 1000 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000) - INSERT INTO ints SELECT 20000 + number % 100 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000) - INSERT INTO ints SELECT 30000 + number % 10 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000) - INSERT INTO ints SELECT 40000 + number % 1 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(10000) + INSERT INTO ints SELECT number AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(5000) + INSERT INTO ints SELECT 10000 + number % 1000 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(5000) + INSERT INTO ints SELECT 20000 + number % 100 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(5000) + INSERT INTO ints SELECT 30000 + number % 10 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(5000) + INSERT INTO ints SELECT 40000 + number % 1 AS i64, i64 AS i32, i64 AS i16, i64 AS i8 FROM numbers(5000) SELECT COUNT() FROM ints l ANY LEFT JOIN ints r USING i64 WHERE i32 = 200042 SELECT COUNT() FROM ints l ANY LEFT JOIN ints r USING i64,i32,i16,i8 WHERE i32 = 200042 From 90487058c42eaef00ac96aa4d639f00cb112fbca Mon Sep 17 00:00:00 2001 From: chertus Date: Wed, 17 Jul 2019 18:16:28 +0300 Subject: [PATCH 32/64] remove wrong infinite loop ckeck --- dbms/src/Common/MemoryTracker.cpp | 8 +------- dbms/src/Common/MemoryTracker.h | 2 -- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/dbms/src/Common/MemoryTracker.cpp b/dbms/src/Common/MemoryTracker.cpp index 91309af1d6c..5a8e6118030 100644 --- a/dbms/src/Common/MemoryTracker.cpp +++ b/dbms/src/Common/MemoryTracker.cpp @@ -117,12 +117,7 @@ void MemoryTracker::alloc(Int64 size) message << " exceeded: would use " << formatReadableSizeWithBinarySuffix(will_be) << " (attempt to allocate chunk of " << size << " bytes)" << ", maximum: " << formatReadableSizeWithBinarySuffix(current_limit); -#ifndef NDEBUG - /// Prevent infinite loop in case of catching & new in cycle. - if (last_attempt_failed) - std::abort(); -#endif - ++last_attempt_failed; + throw DB::Exception(message.str(), DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED); } @@ -137,7 +132,6 @@ void MemoryTracker::alloc(Int64 size) if (auto loaded_next = parent.load(std::memory_order_relaxed)) loaded_next->alloc(size); - last_attempt_failed = 0; } diff --git a/dbms/src/Common/MemoryTracker.h b/dbms/src/Common/MemoryTracker.h index 73f46a447c7..4ce0ac262fa 100644 --- a/dbms/src/Common/MemoryTracker.h +++ b/dbms/src/Common/MemoryTracker.h @@ -115,8 +115,6 @@ public: /// To be able to temporarily stop memory tracker DB::SimpleActionBlocker blocker; - - UInt32 last_attempt_failed = 0; }; From 44db7badbebd17e0b99cbe4a770f7d39cd8d8074 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 17 Jul 2019 18:36:28 +0300 Subject: [PATCH 33/64] done --- dbms/src/Interpreters/AsynchronousMetrics.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dbms/src/Interpreters/AsynchronousMetrics.cpp b/dbms/src/Interpreters/AsynchronousMetrics.cpp index 28a6d5dfb4d..3d09c95e953 100644 --- a/dbms/src/Interpreters/AsynchronousMetrics.cpp +++ b/dbms/src/Interpreters/AsynchronousMetrics.cpp @@ -159,10 +159,14 @@ void AsynchronousMetrics::update() size_t max_part_count_for_partition = 0; + size_t number_of_databases = databases.size(); + size_t total_number_of_tables = 0; + for (const auto & db : databases) { for (auto iterator = db.second->getIterator(context); iterator->isValid(); iterator->next()) { + ++total_number_of_tables; auto & table = iterator->table(); StorageMergeTree * table_merge_tree = dynamic_cast(table.get()); StorageReplicatedMergeTree * table_replicated_merge_tree = dynamic_cast(table.get()); @@ -213,6 +217,9 @@ void AsynchronousMetrics::update() set("ReplicasMaxRelativeDelay", max_relative_delay); set("MaxPartCountForPartition", max_part_count_for_partition); + + set("NumberOfDatabases", number_of_databases); + set("TotalNumberOfTables", total_number_of_tables); } #if USE_TCMALLOC From af44506ab46937b77ed19e2e70e505db6e2510b8 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Wed, 17 Jul 2019 18:45:32 +0300 Subject: [PATCH 34/64] Do not show virtual columns in DESCRIBE TABLE --- dbms/src/Interpreters/InterpreterDescribeQuery.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp index d037f87a857..bf830b697d5 100644 --- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp @@ -101,6 +101,9 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() for (const auto & column : columns) { + if (column.is_virtual) + continue; + res_columns[0]->insert(column.name); res_columns[1]->insert(column.type->getName()); From dd978781079fa849bab85c39cc42cad50adbd407 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 17 Jul 2019 20:19:55 +0300 Subject: [PATCH 35/64] Style --- dbms/src/Interpreters/InterpreterDescribeQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp index bf830b697d5..e6da9b63d7b 100644 --- a/dbms/src/Interpreters/InterpreterDescribeQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDescribeQuery.cpp @@ -103,7 +103,7 @@ BlockInputStreamPtr InterpreterDescribeQuery::executeImpl() { if (column.is_virtual) continue; - + res_columns[0]->insert(column.name); res_columns[1]->insert(column.type->getName()); From dc8b83c882f3f7d8795ffeb8c9d4b99c384eb0bb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 17 Jul 2019 20:23:06 +0300 Subject: [PATCH 36/64] Added a test #6040 --- .../00972_desc_table_virtual_columns.reference | 1 + .../0_stateless/00972_desc_table_virtual_columns.sql | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00972_desc_table_virtual_columns.reference create mode 100644 dbms/tests/queries/0_stateless/00972_desc_table_virtual_columns.sql diff --git a/dbms/tests/queries/0_stateless/00972_desc_table_virtual_columns.reference b/dbms/tests/queries/0_stateless/00972_desc_table_virtual_columns.reference new file mode 100644 index 00000000000..b244da814f9 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00972_desc_table_virtual_columns.reference @@ -0,0 +1 @@ +x UInt64 diff --git a/dbms/tests/queries/0_stateless/00972_desc_table_virtual_columns.sql b/dbms/tests/queries/0_stateless/00972_desc_table_virtual_columns.sql new file mode 100644 index 00000000000..920025a844d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00972_desc_table_virtual_columns.sql @@ -0,0 +1,9 @@ +-- No virtual columns should be output in DESC TABLE query. + +DROP TABLE IF EXISTS upyachka; +CREATE TABLE upyachka (x UInt64) ENGINE = Memory; + +-- Merge table has virtual column `_table` +DESC TABLE merge(currentDatabase(), 'upyachka'); + +DROP TABLE upyachka; From 0f15c01c6802f7ce1a1494c12c846be8c98944cd Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 17 Jul 2019 21:54:34 +0300 Subject: [PATCH 37/64] Added a notion of obsolete settings for our best clients --- dbms/src/Core/Settings.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbms/src/Core/Settings.h b/dbms/src/Core/Settings.h index e06e953810b..b1182cae9bf 100644 --- a/dbms/src/Core/Settings.h +++ b/dbms/src/Core/Settings.h @@ -332,7 +332,11 @@ struct Settings : public SettingsCollection M(SettingBool, allow_simdjson, true, "Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used.") \ \ M(SettingUInt64, max_partitions_per_insert_block, 100, "Limit maximum number of partitions in single INSERTed block. Zero means unlimited. Throw exception if the block contains too many partitions. This setting is a safety threshold, because using large number of partitions is a common misconception.") \ - M(SettingBool, check_query_single_value_result, true, "Return check query result as single 1/0 value") + M(SettingBool, check_query_single_value_result, true, "Return check query result as single 1/0 value") \ + \ + /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ + \ + M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13") DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS) From aaee4724c3ccbb943d6b77c09b10cdac5ea3f11f Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 17 Jul 2019 22:34:29 +0300 Subject: [PATCH 38/64] Update AsynchronousMetrics.cpp --- dbms/src/Interpreters/AsynchronousMetrics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/AsynchronousMetrics.cpp b/dbms/src/Interpreters/AsynchronousMetrics.cpp index 3d09c95e953..621c27d6be8 100644 --- a/dbms/src/Interpreters/AsynchronousMetrics.cpp +++ b/dbms/src/Interpreters/AsynchronousMetrics.cpp @@ -219,7 +219,7 @@ void AsynchronousMetrics::update() set("MaxPartCountForPartition", max_part_count_for_partition); set("NumberOfDatabases", number_of_databases); - set("TotalNumberOfTables", total_number_of_tables); + set("NumberOfTables", total_number_of_tables); } #if USE_TCMALLOC From afa2bd6dfb9c56ad3265ab050fb7c8f6b0450e7d Mon Sep 17 00:00:00 2001 From: chertus Date: Thu, 18 Jul 2019 01:48:31 +0300 Subject: [PATCH 39/64] allow alloc 4Mb more after out-of-limit exception --- dbms/src/Common/MemoryTracker.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbms/src/Common/MemoryTracker.cpp b/dbms/src/Common/MemoryTracker.cpp index 5a8e6118030..b3d661d95ee 100644 --- a/dbms/src/Common/MemoryTracker.cpp +++ b/dbms/src/Common/MemoryTracker.cpp @@ -206,8 +206,11 @@ namespace CurrentMemoryTracker untracked += size; if (untracked > untracked_memory_limit) { - memory_tracker->alloc(untracked); + /// Zero untracked before track. If tracker throws out-of-limit we would be able to alloc up to untracked_memory_limit bytes + /// more. It could be usefull for enlarge Exception message in rethrow logic. + Int64 tmp = untracked; untracked = 0; + memory_tracker->alloc(tmp); } } } From 2f68aa72074663b253fda46f29b9f20bf7cbb793 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Thu, 18 Jul 2019 09:55:05 +0300 Subject: [PATCH 40/64] Fix build with external libcxx (#6010) * First attempt to fix build with external libcxx * Fix build --- CMakeLists.txt | 6 +++++- cmake/find_cxx.cmake | 23 +++++++++++++---------- cmake/find_cxxabi.cmake | 22 ---------------------- contrib/CMakeLists.txt | 5 +---- 4 files changed, 19 insertions(+), 37 deletions(-) delete mode 100644 cmake/find_cxxabi.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 6262d17f2d2..9509a2f9db6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -220,7 +220,6 @@ endif () set(THREADS_PREFER_PTHREAD_FLAG ON) find_package (Threads) -include (cmake/find_cxxabi.cmake) include (cmake/find_cxx.cmake) include (cmake/test_compiler.cmake) @@ -404,6 +403,11 @@ if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32) option (NO_WERROR "Disable -Werror compiler option" ON) endif () +if (USE_LIBCXX) + set (HAVE_LIBCXX 1) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") +endif() + if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostdinc++ -isystem ${LIBCXX_INCLUDE_DIR} -isystem ${LIBCXXABI_INCLUDE_DIR}") endif () diff --git a/cmake/find_cxx.cmake b/cmake/find_cxx.cmake index a10c3259ad9..2b2952f6efd 100644 --- a/cmake/find_cxx.cmake +++ b/cmake/find_cxx.cmake @@ -1,23 +1,26 @@ if (NOT APPLE) - option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx library instead of bundled" ${NOT_UNBUNDLED}) + option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled" ${NOT_UNBUNDLED}) endif () if (USE_INTERNAL_LIBCXX_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/include/vector") - message (WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive") - set (USE_INTERNAL_LIBCXX_LIBRARY 0) + message (WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive") + set (USE_INTERNAL_LIBCXX_LIBRARY 0) +endif () + +if (USE_INTERNAL_LIBCXX_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxxabi/src") + message (WARNING "submodule contrib/libcxxabi is missing. to fix try run: \n git submodule update --init --recursive") + set (USE_INTERNAL_LIBCXXABI_LIBRARY 0) endif () if (NOT USE_INTERNAL_LIBCXX_LIBRARY) find_library (LIBCXX_LIBRARY c++) - find_path (LIBCXX_INCLUDE_DIR NAMES vector PATHS ${LIBCXX_INCLUDE_PATHS}) -endif () - -if (LIBCXX_LIBRARY AND LIBCXX_INCLUDE_DIR) + find_library (LIBCXXABI_LIBRARY c++abi) else () set (LIBCXX_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxx/include) - set (USE_INTERNAL_LIBCXX_LIBRARY 1) + set (LIBCXXABI_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxxabi/include) set (LIBCXX_LIBRARY cxx_static) - set (HAVE_LIBCXX 1) + set (LIBCXXABI_LIBRARY cxxabi_static) endif () -message (STATUS "Using libcxx: ${LIBCXX_INCLUDE_DIR} : ${LIBCXX_LIBRARY}") +message (STATUS "Using libcxx: ${LIBCXX_LIBRARY}") +message (STATUS "Using libcxxabi: ${LIBCXXABI_LIBRARY}") diff --git a/cmake/find_cxxabi.cmake b/cmake/find_cxxabi.cmake deleted file mode 100644 index 8f240597bc4..00000000000 --- a/cmake/find_cxxabi.cmake +++ /dev/null @@ -1,22 +0,0 @@ -if (NOT APPLE) - option (USE_INTERNAL_LIBCXXABI_LIBRARY "Set to FALSE to use system libcxxabi library instead of bundled" ${NOT_UNBUNDLED}) -endif () - -if (USE_INTERNAL_LIBCXXABI_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxxabi/src") - message (WARNING "submodule contrib/libcxxabi is missing. to fix try run: \n git submodule update --init --recursive") - set (USE_INTERNAL_LIBCXXABI_LIBRARY 0) -endif () - -if (NOT USE_INTERNAL_LIBCXXABI_LIBRARY) - find_library (LIBCXXABI_LIBRARY cxxabi) - find_path (LIBCXXABI_INCLUDE_DIR NAMES vector PATHS ${LIBCXXABI_INCLUDE_PATHS}) -endif () - -if (LIBCXXABI_LIBRARY AND LIBCXXABI_INCLUDE_DIR) -else () - set (LIBCXXABI_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcxxabi/include) - set (USE_INTERNAL_LIBCXXABI_LIBRARY 1) - set (LIBCXXABI_LIBRARY cxxabi_static) -endif () - -message (STATUS "Using libcxxabi: ${LIBCXXABI_INCLUDE_DIR} : ${LIBCXXABI_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1280a309b01..2cc8ae37806 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -15,12 +15,9 @@ if (USE_INTERNAL_UNWIND_LIBRARY) add_subdirectory (libunwind-cmake) endif () -if (USE_LIBCXX AND USE_INTERNAL_LIBCXXABI_LIBRARY) - add_subdirectory(libcxxabi-cmake) -endif() - if (USE_LIBCXX AND USE_INTERNAL_LIBCXX_LIBRARY) add_subdirectory(libcxx-cmake) + add_subdirectory(libcxxabi-cmake) endif() From 83c8d4872d71456a91a99270233d7a21e24b8cdf Mon Sep 17 00:00:00 2001 From: BayoNet Date: Thu, 18 Jul 2019 10:56:16 +0300 Subject: [PATCH 41/64] DOCAPI-7400: Gorilla codecs. Basic review of English text. (#5904) Gorilla codecs. Rewriting of English text. Update of other codecs description. --- docs/en/query_language/create.md | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/en/query_language/create.md b/docs/en/query_language/create.md index a6ae57bc00f..94d32cc49f5 100644 --- a/docs/en/query_language/create.md +++ b/docs/en/query_language/create.md @@ -107,22 +107,26 @@ Besides default data compression, defined in [server settings](../operations/ser Supported compression algorithms: -- `NONE` - no compression for data applied -- `LZ4` -- `LZ4HC(level)` - (level) - LZ4\_HC compression algorithm with defined level. -Possible `level` range: \[3, 12\]. Default value: 9. Greater values stands for better compression and higher CPU usage. Recommended value range: [4,9]. -- `ZSTD(level)` - ZSTD compression algorithm with defined `level`. Possible `level` value range: \[1, 22\]. Default value: 1. -Greater values stands for better compression and higher CPU usage. -- `Delta(delta_bytes)` - compression approach when raw values are replace with difference of two neighbour values. Up to `delta_bytes` are used for storing delta value. -Possible `delta_bytes` values: 1, 2, 4, 8. Default value for delta bytes is `sizeof(type)`, if it is equals to 1, 2, 4, 8 and equals to 1 otherwise. -- `DoubleDelta` - stores delta of deltas in compact binary form, compressing values down to 1 bit (in the best case). Best compression rates are achieved on monotonic sequences with constant stride, e.g. time samples. Can be used against any fixed-width type. Implementation is based on [Gorilla paper](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf), and extended to support 64bit types. The drawback is 1 extra bit for 32-byte wide deltas: 5-bit prefix instead of 4-bit prefix. -- `Gorilla` - stores (parts of) xored values in compact binary form, compressing values down to 1 bit (in the best case). Best compression rate is achieved when neighbouring values are binary equal. Basic use case - floating point data that do not change rapidly. Implementation is based on [Gorilla paper](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf), and extended to support 64bit types. +- `NONE` — No compression. +- `LZ4` — Lossless [data compression algorithm](https://github.com/lz4/lz4) used by default. Applies LZ4 fast compression. +- `LZ4HC[(level)]` — LZ4 CH (high compression) algorithm with configurable level. Default level: 9. If you set `level <= 0`, the default level is applied. Possible levels: [1, 12]. Recommended levels are in range: [4, 9]. +- `ZSTD[(level)]` — [ZSTD compression algorithm](https://en.wikipedia.org/wiki/Zstandard) with configurable `level`. Possible levels: [1, 22]. Default value: 1. +- `Delta(delta_bytes)` — compression approach, when raw values are replaced with the difference of two neighbour values. Up to `delta_bytes` are used for storing delta value, so `delta_bytes` is a maximum size of raw values. +Possible `delta_bytes` values: 1, 2, 4, 8. Default value for `delta_bytes` is `sizeof(type)`, if it is equals to 1, 2, 4, 8. Otherwise it equals 1. +- `DoubleDelta` — Compresses values down to 1 bit (in the best case), using deltas calculation. Best compression rates are achieved on monotonic sequences with constant stride, for example, time series data. Can be used with any fixed-width type. Implements the algorithm used in Gorilla TSDB, extending it to support 64 bit types. Uses 1 extra bit for 32 byte deltas: 5 bit prefix instead of 4 bit prefix. For additional information, see the "Compressing time stamps" section of the [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf) document. +- `Gorilla` — Compresses values down to 1 bit (in the best case). The codec is efficient when storing series of floating point values that change slowly, because the best compression rate is achieved when neighbouring values are binary equal. Implements the algorithm used in Gorilla TSDB, extending it to support 64 bit types. For additional information, see the "Compressing values" section of the [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf) document. + +High compression levels useful for asymmetric scenarios, like compress once, decompress a lot of times. Greater levels stands for better compression and higher CPU usage. + +!!!warning + You cannot decompress ClickHouse database files with external utilities, for example, `lz4`. Use the special utility [clickhouse-compressor](https://github.com/yandex/ClickHouse/tree/master/dbms/programs/compressor). Syntax example: + ``` CREATE TABLE codec_example ( - dt Date CODEC(ZSTD), /* используется уровень сжатия по-умолчанию */ + dt Date CODEC(ZSTD), ts DateTime CODEC(LZ4HC), float_value Float32 CODEC(NONE), double_value Float64 CODEC(LZ4HC(9)) @@ -134,6 +138,7 @@ ORDER BY dt Codecs can be combined in a pipeline. Default table codec is not included into pipeline (if it should be applied to a column, you have to specify it explicitly in pipeline). Example below shows an optimization approach for storing timeseries metrics. Usually, values for particular metric, stored in `path` does not differ significantly from point to point. Using delta-encoding allows to reduce disk space usage significantly. + ``` CREATE TABLE timeseries_example ( From d48995f1e74732fbd8b62441c865732445409c2e Mon Sep 17 00:00:00 2001 From: BayoNet Date: Thu, 18 Jul 2019 14:04:45 +0300 Subject: [PATCH 42/64] DOCAPI-7444: RU <-> EN docs regular sync. (#5944) * DOCAPI-7444: RU <-> EN docs regular sync. --- .../en/operations/server_settings/settings.md | 2 +- docs/en/operations/table_engines/file.md | 2 +- .../agg_functions/combinators.md | 14 ++++++-- docs/en/query_language/functions/geo.md | 32 ++++++++++++++++++ .../functions/other_functions.md | 27 +++++++++++++-- .../example_datasets/star_schema.md | 10 +++--- docs/ru/interfaces/formats.md | 2 ++ .../ru/operations/server_settings/settings.md | 2 +- docs/ru/operations/settings/settings.md | 15 +++------ docs/ru/operations/table_engines/file.md | 2 +- docs/ru/operations/table_engines/kafka.md | 4 +-- .../agg_functions/combinators.md | 15 +++++++-- docs/ru/query_language/functions/geo.md | 23 ++++++++----- .../functions/other_functions.md | 33 +++++++++++++++++++ .../functions/other_functions.md | 2 +- 15 files changed, 145 insertions(+), 40 deletions(-) diff --git a/docs/en/operations/server_settings/settings.md b/docs/en/operations/server_settings/settings.md index 493a558c8e2..2f87233e6a3 100644 --- a/docs/en/operations/server_settings/settings.md +++ b/docs/en/operations/server_settings/settings.md @@ -514,7 +514,7 @@ Use the following parameters to configure logging: ``` -## path +## path {#server_settings-path} The path to the directory containing data. diff --git a/docs/en/operations/table_engines/file.md b/docs/en/operations/table_engines/file.md index 82c536646ac..9e3c5b3400b 100644 --- a/docs/en/operations/table_engines/file.md +++ b/docs/en/operations/table_engines/file.md @@ -71,7 +71,7 @@ $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64 ## Details of Implementation -- Multiple SELECT queries can be performed concurrently, but INSERT queries will wait each other. +- Multiple `SELECT` queries can be performed concurrently, but `INSERT` queries will wait each other. - Not supported: - `ALTER` - `SELECT ... SAMPLE` diff --git a/docs/en/query_language/agg_functions/combinators.md b/docs/en/query_language/agg_functions/combinators.md index 852b396332c..133b4d489d7 100644 --- a/docs/en/query_language/agg_functions/combinators.md +++ b/docs/en/query_language/agg_functions/combinators.md @@ -22,13 +22,21 @@ Example 2: `uniqArray(arr)` – Count the number of unique elements in all 'arr' ## -State -If you apply this combinator, the aggregate function doesn't return the resulting value (such as the number of unique values for the `uniq` function), but an intermediate state of the aggregation (for `uniq`, this is the hash table for calculating the number of unique values). This is an AggregateFunction(...) that can be used for further processing or stored in a table to finish aggregating later. To work with these states, use the [AggregatingMergeTree](../../operations/table_engines/aggregatingmergetree.md) table engine, the functions [`finalizeAggregation`](../functions/other_functions.md#finalizeaggregation) and [`runningAccumulate`](../functions/other_functions.md#function-runningaccumulate), and the combinators -Merge and -MergeState described below. +If you apply this combinator, the aggregate function doesn't return the resulting value (such as the number of unique values for the [uniq](reference.md#agg_function-uniq) function), but an intermediate state of the aggregation (for `uniq`, this is the hash table for calculating the number of unique values). This is an `AggregateFunction(...)` that can be used for further processing or stored in a table to finish aggregating later. -## -Merge +To work with these states, use: + +- [AggregatingMergeTree](../../operations/table_engines/aggregatingmergetree.md) table engine. +- [finalizeAggregation](../functions/other_functions.md#function-finalizeaggregation) function. +- [runningAccumulate](../functions/other_functions.md#function-runningaccumulate) function. +- [-Merge](#aggregate_functions_combinators_merge) combinator. +- [-MergeState](#aggregate_functions_combinators_mergestate) combinator. + +## -Merge {#aggregate_functions_combinators_merge} If you apply this combinator, the aggregate function takes the intermediate aggregation state as an argument, combines the states to finish aggregation, and returns the resulting value. -## -MergeState. +## -MergeState {#aggregate_functions_combinators_mergestate} Merges the intermediate aggregation states in the same way as the -Merge combinator. However, it doesn't return the resulting value, but an intermediate aggregation state, similar to the -State combinator. diff --git a/docs/en/query_language/functions/geo.md b/docs/en/query_language/functions/geo.md index 97c4d1b5b4f..2c84a4516ba 100644 --- a/docs/en/query_language/functions/geo.md +++ b/docs/en/query_language/functions/geo.md @@ -151,4 +151,36 @@ SELECT geohashDecode('ezs42') AS res └─────────────────────────────────┘ ``` +## geoToH3 + +Calculates [H3](https://uber.github.io/h3/#/documentation/overview/introduction) point index `(lon, lat)` with specified resolution. + +``` +geoToH3(lon, lat, resolution) +``` + +**Input values** + +- `lon` — Longitude. Type: [Float64](../../data_types/float.md). +- `lat` — Latitude. Type: [Float64](../../data_types/float.md). +- `resolution` — Index resolution. Range: `[0, 15]`. Type: [UInt8](../../data_types/int_uint.md). + +**Returned values** + +- Hexagon index number. +- 0 in case of error. + +Type: [UInt64](../../data_types/int_uint.md). + +**Example** + +``` sql +SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index +``` +``` +┌────────────h3Index─┐ +│ 644325524701193974 │ +└────────────────────┘ +``` + [Original article](https://clickhouse.yandex/docs/en/query_language/functions/geo/) diff --git a/docs/en/query_language/functions/other_functions.md b/docs/en/query_language/functions/other_functions.md index d3b7b2082d4..007f1352775 100644 --- a/docs/en/query_language/functions/other_functions.md +++ b/docs/en/query_language/functions/other_functions.md @@ -627,15 +627,36 @@ SELECT replicate(1, ['a', 'b', 'c']) └───────────────────────────────┘ ``` -## filesystemAvailable +## filesystemAvailable {#function-filesystemavailable} -Returns the remaining space information of the disk, in bytes. This information is evaluated using the configured by path. +Returns the amount of remaining space in the filesystem where the files of the databases located. See the [path](../../operations/server_settings/settings.md#server_settings-path) server setting description. + +``` +filesystemAvailable() +``` + +**Returned values** + +- Amount of remaining space in bytes. + +Type: [UInt64](../../data_types/int_uint.md). + +**Example** + +```sql +SELECT filesystemAvailable() AS "Free space", toTypeName(filesystemAvailable()) AS "Type" +``` +```text +┌──Free space─┬─Type───┐ +│ 18152624128 │ UInt64 │ +└─────────────┴────────┘ +``` ## filesystemCapacity Returns the capacity information of the disk, in bytes. This information is evaluated using the configured by path. -## finalizeAggregation +## finalizeAggregation {#function-finalizeaggregation} Takes state of aggregate function. Returns result of aggregation (finalized state). diff --git a/docs/ru/getting_started/example_datasets/star_schema.md b/docs/ru/getting_started/example_datasets/star_schema.md index 1d8af3b29a5..545eaeea6a6 100644 --- a/docs/ru/getting_started/example_datasets/star_schema.md +++ b/docs/ru/getting_started/example_datasets/star_schema.md @@ -101,11 +101,11 @@ CREATE TABLE lineorder_flat ENGINE = MergeTree PARTITION BY toYear(LO_ORDERDATE) ORDER BY (LO_ORDERDATE, LO_ORDERKEY) AS -SELECT * -FROM lineorder -ANY INNER JOIN customer ON LO_CUSTKEY = C_CUSTKEY -ANY INNER JOIN supplier ON LO_SUPPKEY = S_SUPPKEY -ANY INNER JOIN part ON LO_PARTKEY = P_PARTKEY; +SELECT l.*, c.*, s.*, p.* +FROM lineorder l + ANY INNER JOIN customer c ON (c.C_CUSTKEY = l.LO_CUSTKEY) + ANY INNER JOIN supplier s ON (s.S_SUPPKEY = l.LO_SUPPKEY) + ANY INNER JOIN part p ON (p.P_PARTKEY = l.LO_PARTKEY); ALTER TABLE lineorder_flat DROP COLUMN C_CUSTKEY, DROP COLUMN S_SUPPKEY, DROP COLUMN P_PARTKEY; ``` diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index c02ed3d4993..bc685443b0d 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -165,6 +165,8 @@ clickhouse-client --format_csv_delimiter="|" --query="INSERT INTO test.csv FORMA При парсинге, все значения могут парситься как в кавычках, так и без кавычек. Поддерживаются как двойные, так и одинарные кавычки. Строки также могут быть без кавычек. В этом случае они парсятся до символа-разделителя или перевода строки (CR или LF). В нарушение RFC, в случае парсинга строк не в кавычках, начальные и конечные пробелы и табы игнорируются. В качестве перевода строки, поддерживаются как Unix (LF), так и Windows (CR LF) и Mac OS Classic (LF CR) варианты. +Если установлена настройка [input_format_defaults_for_omitted_fields = 1](../operations/settings/settings.md#session_settings-input_format_defaults_for_omitted_fields), то пустые значения без кавычек заменяются значениями по умолчанию для типа данных столбца. + `NULL` форматируется в виде `\N`. Формат CSV поддерживает вывод totals и extremes аналогично `TabSeparated`. diff --git a/docs/ru/operations/server_settings/settings.md b/docs/ru/operations/server_settings/settings.md index 8f1bbcb7488..c1444668bb2 100644 --- a/docs/ru/operations/server_settings/settings.md +++ b/docs/ru/operations/server_settings/settings.md @@ -514,7 +514,7 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ``` -## path +## path {#server_settings-path} Путь к каталогу с данными. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index e320231d397..fa3c67c687c 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -181,20 +181,15 @@ Ok. ## input_format_defaults_for_omitted_fields {#session_settings-input_format_defaults_for_omitted_fields} -Включает/выключает расширенный обмен данными между клиентом ClickHouse и сервером ClickHouse. Параметр применяется для запросов `INSERT`. +При вставке данных запросом `INSERT`, заменяет пропущенные поля значениям по умолчанию для типа данных столбца. -При выполнении запроса`INSERT`, клиент ClickHouse подготавливает данные и отправляет их на сервер для записи. При подготовке данных клиент получает структуру таблицы от сервера. В некоторых случаях клиенту требуется больше информации, чем сервер отправляет по умолчанию. Включите расширенный обмен данными с помощью настройки `input_format_defaults_for_omitted_fields = 1`. +Поддерживаемые форматы вставки: -Если расширенный обмен данными включен, сервер отправляет дополнительные метаданные вместе со структурой таблицы. Состав метаданных зависит от операции. - -Операции, для которых может потребоваться включить расширенный обмен данными: - -- Вставка данных в формате [JSONEachRow](../../interfaces/formats.md#jsoneachrow). - -Для всех остальных операций ClickHouse не применяет этот параметр. +- [JSONEachRow](../../interfaces/formats.md#jsoneachrow) +- [CSV](../../interfaces/formats.md#csv) !!! note "Примечание" - Функциональность расширенного обмена данными потребляет дополнительные вычислительные ресурсы на сервере и может снизить производительность. + Когда опция включена, сервер отправляет клиенту расширенные метаданные. Это требует дополнительных вычислительных ресурсов на сервере и может снизить производительность. Возможные значения: diff --git a/docs/ru/operations/table_engines/file.md b/docs/ru/operations/table_engines/file.md index 731204f928a..b67823b988a 100644 --- a/docs/ru/operations/table_engines/file.md +++ b/docs/ru/operations/table_engines/file.md @@ -68,7 +68,7 @@ $ echo -e "1,2\n3,4" | clickhouse-local -q "CREATE TABLE table (a Int64, b Int64 ## Детали реализации -- Поддерживается многопоточное чтение и однопоточная запись. +- Поддерживается одновременное выполнение множества запросов `SELECT`, запросы `INSERT` могут выполняться только последовательно. - Не поддерживается: - использование операций `ALTER` и `SELECT...SAMPLE`; - индексы; diff --git a/docs/ru/operations/table_engines/kafka.md b/docs/ru/operations/table_engines/kafka.md index 3fe2e4d5cba..086d4fb4f08 100644 --- a/docs/ru/operations/table_engines/kafka.md +++ b/docs/ru/operations/table_engines/kafka.md @@ -25,7 +25,7 @@ SETTINGS [kafka_row_delimiter = 'delimiter_symbol',] [kafka_schema = '',] [kafka_num_consumers = N,] - [kafka_skip_broken_messages = <0|1>] + [kafka_skip_broken_messages = N] ``` Обязательные параметры: @@ -40,7 +40,7 @@ SETTINGS - `kafka_row_delimiter` – символ-разделитель записей (строк), которым завершается сообщение. - `kafka_schema` – опциональный параметр, необходимый, если используется формат, требующий определения схемы. Например, [Cap'n Proto](https://capnproto.org/) требует путь к файлу со схемой и название корневого объекта `schema.capnp:Message`. - `kafka_num_consumers` – количество потребителей (consumer) на таблицу. По умолчанию: `1`. Укажите больше потребителей, если пропускная способность одного потребителя недостаточна. Общее число потребителей не должно превышать количество партиций в топике, так как на одну партицию может быть назначено не более одного потребителя. -- `kafka_skip_broken_messages` – режим обработки сообщений Kafka. Если `kafka_skip_broken_messages = 1`, то движок отбрасывает сообщения Кафки, которые не получилось обработать. Одно сообщение в точности соответствует одной записи (строке). +- `kafka_skip_broken_messages` – максимальное количество некорректных сообщений в блоке. Если `kafka_skip_broken_messages = N`, то движок отбрасывает `N` сообщений Кафки, которые не получилось обработать. Одно сообщение в точности соответствует одной записи (строке). Значение по умолчанию – 0. Примеры diff --git a/docs/ru/query_language/agg_functions/combinators.md b/docs/ru/query_language/agg_functions/combinators.md index dfee76bb79d..1fcdb111e17 100644 --- a/docs/ru/query_language/agg_functions/combinators.md +++ b/docs/ru/query_language/agg_functions/combinators.md @@ -23,13 +23,22 @@ ## -State -В случае применения этого комбинатора, агрегатная функция возвращает не готовое значение (например, в случае функции `uniq` — количество уникальных значений), а промежуточное состояние агрегации (например, в случае функции `uniq` — хэш-таблицу для расчёта количества уникальных значений), которое имеет тип AggregateFunction(...) и может использоваться для дальнейшей обработки или может быть сохранено в таблицу для последующей доагрегации - смотрите разделы «AggregatingMergeTree» и «функции для работы с промежуточными состояниями агрегации». +В случае применения этого комбинатора, агрегатная функция возвращает не готовое значение (например, в случае функции [uniq](reference.md#agg_function-uniq) — количество уникальных значений), а промежуточное состояние агрегации (например, в случае функции `uniq` — хэш-таблицу для расчёта количества уникальных значений), которое имеет тип `AggregateFunction(...)` и может использоваться для дальнейшей обработки или может быть сохранено в таблицу для последующей доагрегации. -## -Merge +Для работы с промежуточными состояниями предназначены: + +- Движок таблиц [AggregatingMergeTree](../../operations/table_engines/aggregatingmergetree.md). +- Функция [finalizeAggregation](../functions/other_functions.md#function-finalizeaggregation). +- Функция [runningAccumulate](../functions/other_functions.md#function-runningaccumulate). +- Комбинатор [-Merge](#aggregate_functions_combinators_merge). +- Комбинатор [-MergeState](#aggregate_functions_combinators_mergestate). + + +## -Merge {#aggregate_functions_combinators_merge} В случае применения этого комбинатора, агрегатная функция будет принимать в качестве аргумента промежуточное состояние агрегации, доагрегировать (объединять вместе) эти состояния, и возвращать готовое значение. -## -MergeState. +## -MergeState {#aggregate_functions_combinators_mergestate} Выполняет слияние промежуточных состояний агрегации, аналогично комбинатору -Merge, но возвращает не готовое значение, а промежуточное состояние агрегации, аналогично комбинатору -State. diff --git a/docs/ru/query_language/functions/geo.md b/docs/ru/query_language/functions/geo.md index 33092cf804b..b8e37c15aca 100644 --- a/docs/ru/query_language/functions/geo.md +++ b/docs/ru/query_language/functions/geo.md @@ -132,13 +132,17 @@ SELECT geohashEncode(-5.60302734375, 42.593994140625, 0) AS res Декодирует любую строку, закодированную в geohash, на долготу и широту. +``` +geohashDecode(geohash_string) +``` + **Входные значения** -- encoded string — строка, содержащая geohash. +- `geohash_string` — строка, содержащая geohash. **Возвращаемые значения** -- (longitude, latitude) — широта и долгота. Кортеж из двух значений типа `Float64`. +- `(longitude, latitude)` — широта и долгота. Кортеж из двух значений типа `Float64`. **Пример** @@ -154,7 +158,7 @@ SELECT geohashDecode('ezs42') AS res ## geoToH3 -Получает H3 индекс точки (lon, lat) с заданным разрешением +Получает H3 индекс точки `(lon, lat)` с заданным разрешением ``` geoToH3(lon, lat, resolution) @@ -162,15 +166,16 @@ geoToH3(lon, lat, resolution) **Входные значения** -- `lon` - географическая долгота. Тип данных — [Float64](../../data_types/float.md). -- `lat` - географическая широта. Тип данных — [Float64](../../data_types/float.md). -- `resolution` - требуемое разрешение индекса. Тип данных — [UInt8](../../data_types/int_uint.md). Диапазон возможных значение — `[0, 15]`. +- `lon` — географическая долгота. Тип данных — [Float64](../../data_types/float.md). +- `lat` — географическая широта. Тип данных — [Float64](../../data_types/float.md). +- `resolution` — требуемое разрешение индекса. Тип данных — [UInt8](../../data_types/int_uint.md). Диапазон возможных значений — `[0, 15]`. **Возвращаемые значения** -Возвращает значение с типом [UInt64] (../../data_types/int_uint.md). -`0` в случае ошибки. -Иначе возвращается индексный номер шестиугольника. +- Порядковый номер шестиугольника. +- 0 в случае ошибки. + +Тип — [UInt64](../../data_types/int_uint.md). **Пример** diff --git a/docs/ru/query_language/functions/other_functions.md b/docs/ru/query_language/functions/other_functions.md index 62af103e02d..6e4913638ee 100644 --- a/docs/ru/query_language/functions/other_functions.md +++ b/docs/ru/query_language/functions/other_functions.md @@ -600,6 +600,39 @@ SELECT replicate(1, ['a', 'b', 'c']) └───────────────────────────────┘ ``` +## filesystemAvailable {#function-filesystemavailable} + +Возвращает объем оставшегося места в файловой системе, в которой расположены файлы баз данных. Смотрите описание конфигурационного параметра сервера [path](../../operations/server_settings/settings.md#server_settings-path). + +``` +filesystemAvailable() +``` + +**Возвращаемое значение** + +- Объем свободного места. + +Тип — [UInt64](../../data_types/int_uint.md). + +**Пример** + +```sql +SELECT filesystemAvailable() AS "Free space", toTypeName(filesystemAvailable()) AS "Type" +``` +```text +┌──Free space─┬─Type───┐ +│ 18152624128 │ UInt64 │ +└─────────────┴────────┘ +``` + +## filesystemCapacity + +Возвращает данные о ёмкости диска. + +## finalizeAggregation {#function-finalizeaggregation} + +Принимает состояние агрегатной функции. Возвращает результат агрегирования. + ## runningAccumulate {#function-runningaccumulate} Принимает на вход состояния агрегатной функции и возвращает столбец со значениями, которые представляют собой результат мёржа этих состояний для выборки строк из блока от первой до текущей строки. Например, принимает состояние агрегатной функции (например, `runningAccumulate(uniqState(UserID))`), и для каждой строки блока возвращает результат агрегатной функции после мёржа состояний функции для всех предыдущих строк и текущей. Таким образом, результат зависит от разбиения данных по блокам и от порядка данных в блоке. diff --git a/docs/zh/query_language/functions/other_functions.md b/docs/zh/query_language/functions/other_functions.md index 85804c0a75d..84fbdaeb3ca 100644 --- a/docs/zh/query_language/functions/other_functions.md +++ b/docs/zh/query_language/functions/other_functions.md @@ -637,7 +637,7 @@ SELECT replicate(1, ['a', 'b', 'c']) 返回磁盘的容量信息，以字节为单位。使用配置文件中的path配置评估此信息。 -## finalizeAggregation +## finalizeAggregation {#function-finalizeaggregation} 获取聚合函数的状态。返回聚合结果（最终状态）。 From 3aaa87171f0f755b32208cddcb5d70ee7feddbbe Mon Sep 17 00:00:00 2001 From: proller Date: Thu, 18 Jul 2019 14:55:14 +0300 Subject: [PATCH 43/64] Debian post install: remove dot from Password for default user. --- debian/clickhouse-server.templates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/clickhouse-server.templates b/debian/clickhouse-server.templates index 3053c18c79f..fdab88cf877 100644 --- a/debian/clickhouse-server.templates +++ b/debian/clickhouse-server.templates @@ -1,3 +1,3 @@ Template: clickhouse-server/default-password Type: password -Description: Password for default user. +Description: Password for default user From 34e1b81f84a456d63ad8c307a0154b92bb83f424 Mon Sep 17 00:00:00 2001 From: chertus Date: Thu, 18 Jul 2019 18:07:41 +0300 Subject: [PATCH 44/64] trying to speedup Allocator::realloc() --- dbms/src/Common/Allocator.h | 133 +++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 61 deletions(-) diff --git a/dbms/src/Common/Allocator.h b/dbms/src/Common/Allocator.h index abaa5927e3d..0efbc410a00 100644 --- a/dbms/src/Common/Allocator.h +++ b/dbms/src/Common/Allocator.h @@ -114,7 +114,76 @@ public: void * alloc(size_t size, size_t alignment = 0) { CurrentMemoryTracker::alloc(size); + return allocNoTrack(size, alignment); + } + /// Free memory range. + void free(void * buf, size_t size) + { + freeNoTrack(buf, size); + CurrentMemoryTracker::free(size); + } + + /** Enlarge memory range. + * Data from old range is moved to the beginning of new range. + * Address of memory range could change. + */ + void * realloc(void * buf, size_t old_size, size_t new_size, size_t alignment = 0) + { + if (old_size == new_size) + { + /// nothing to do. + /// BTW, it's not possible to change alignment while doing realloc. + } + else if (old_size < mmap_threshold && new_size < mmap_threshold && alignment <= MALLOC_MIN_ALIGNMENT) + { + /// Resize malloc'd memory region with no special alignment requirement. + CurrentMemoryTracker::realloc(old_size, new_size); + + void * new_buf = ::realloc(buf, new_size); + if (nullptr == new_buf) + DB::throwFromErrno("Allocator: Cannot realloc from " + formatReadableSizeWithBinarySuffix(old_size) + " to " + formatReadableSizeWithBinarySuffix(new_size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + + buf = new_buf; + if constexpr (clear_memory) + if (new_size > old_size) + memset(reinterpret_cast(buf) + old_size, 0, new_size - old_size); + } + else if (old_size >= mmap_threshold && new_size >= mmap_threshold) + { + /// Resize mmap'd memory region. + CurrentMemoryTracker::realloc(old_size, new_size); + + // On apple and freebsd self-implemented mremap used (common/mremap.h) + buf = clickhouse_mremap(buf, old_size, new_size, MREMAP_MAYMOVE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (MAP_FAILED == buf) + DB::throwFromErrno("Allocator: Cannot mremap memory chunk from " + formatReadableSizeWithBinarySuffix(old_size) + " to " + formatReadableSizeWithBinarySuffix(new_size) + ".", DB::ErrorCodes::CANNOT_MREMAP); + + /// No need for zero-fill, because mmap guarantees it. + } + else + { + /// All other cases that requires a copy. + CurrentMemoryTracker::realloc(old_size, new_size); + + void * new_buf = allocNoTrack(new_size, alignment); + memcpy(new_buf, buf, std::min(old_size, new_size)); + freeNoTrack(buf, old_size); + buf = new_buf; + } + + return buf; + } + +protected: + static constexpr size_t getStackThreshold() + { + return 0; + } + +private: + void * allocNoTrack(size_t size, size_t alignment) + { void * buf; if (size >= mmap_threshold) @@ -149,15 +218,14 @@ public: if (0 != res) DB::throwFromErrno("Cannot allocate memory (posix_memalign) " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY, res); - if (clear_memory) + if constexpr (clear_memory) memset(buf, 0, size); } } return buf; } - /// Free memory range. - void free(void * buf, size_t size) + void freeNoTrack(void * buf, size_t size) { if (size >= mmap_threshold) { @@ -168,63 +236,6 @@ public: { ::free(buf); } - - CurrentMemoryTracker::free(size); - } - - /** Enlarge memory range. - * Data from old range is moved to the beginning of new range. - * Address of memory range could change. - */ - void * realloc(void * buf, size_t old_size, size_t new_size, size_t alignment = 0) - { - if (old_size == new_size) - { - /// nothing to do. - /// BTW, it's not possible to change alignment while doing realloc. - } - else if (old_size < mmap_threshold && new_size < mmap_threshold && alignment <= MALLOC_MIN_ALIGNMENT) - { - /// Resize malloc'd memory region with no special alignment requirement. - CurrentMemoryTracker::realloc(old_size, new_size); - - void * new_buf = ::realloc(buf, new_size); - if (nullptr == new_buf) - DB::throwFromErrno("Allocator: Cannot realloc from " + formatReadableSizeWithBinarySuffix(old_size) + " to " + formatReadableSizeWithBinarySuffix(new_size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - - buf = new_buf; - if (clear_memory && new_size > old_size) - memset(reinterpret_cast(buf) + old_size, 0, new_size - old_size); - } - else if (old_size >= mmap_threshold && new_size >= mmap_threshold) - { - /// Resize mmap'd memory region. - CurrentMemoryTracker::realloc(old_size, new_size); - - // On apple and freebsd self-implemented mremap used (common/mremap.h) - buf = clickhouse_mremap(buf, old_size, new_size, MREMAP_MAYMOVE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (MAP_FAILED == buf) - DB::throwFromErrno("Allocator: Cannot mremap memory chunk from " + formatReadableSizeWithBinarySuffix(old_size) + " to " + formatReadableSizeWithBinarySuffix(new_size) + ".", DB::ErrorCodes::CANNOT_MREMAP); - - /// No need for zero-fill, because mmap guarantees it. - } - else - { - /// All other cases that requires a copy. MemoryTracker is called inside 'alloc', 'free' methods. - - void * new_buf = alloc(new_size, alignment); - memcpy(new_buf, buf, std::min(old_size, new_size)); - free(buf, old_size); - buf = new_buf; - } - - return buf; - } - -protected: - static constexpr size_t getStackThreshold() - { - return 0; } }; @@ -267,7 +278,7 @@ public: { if (size <= N) { - if (Base::clear_memory) + if constexpr (Base::clear_memory) memset(stack_memory, 0, N); return stack_memory; } From d8579714b8ea1a9117816f26a706e1a2293f8024 Mon Sep 17 00:00:00 2001 From: chertus Date: Thu, 18 Jul 2019 18:25:23 +0300 Subject: [PATCH 45/64] trying to speedup Allocator::realloc() (attempt 2) --- dbms/src/Common/Allocator.h | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dbms/src/Common/Allocator.h b/dbms/src/Common/Allocator.h index 0efbc410a00..e9569673678 100644 --- a/dbms/src/Common/Allocator.h +++ b/dbms/src/Common/Allocator.h @@ -108,6 +108,7 @@ class AllocatorWithHint : Hint { protected: static constexpr bool clear_memory = clear_memory_; + static constexpr size_t small_memory_threshold = mmap_threshold; public: /// Allocate memory range. @@ -161,9 +162,9 @@ public: /// No need for zero-fill, because mmap guarantees it. } - else + else if (new_size < small_memory_threshold) { - /// All other cases that requires a copy. + /// Small allocs that requires a copy. Assume there's enough memory in system. Call CurrentMemoryTracker once. CurrentMemoryTracker::realloc(old_size, new_size); void * new_buf = allocNoTrack(new_size, alignment); @@ -171,6 +172,15 @@ public: freeNoTrack(buf, old_size); buf = new_buf; } + else + { + /// Big allocs that requires a copy. MemoryTracker is called inside 'alloc', 'free' methods. + + void * new_buf = alloc(new_size, alignment); + memcpy(new_buf, buf, std::min(old_size, new_size)); + free(buf, old_size); + buf = new_buf; + } return buf; } From 30ca545b09f0c55409acbab0cf6293427d595af9 Mon Sep 17 00:00:00 2001 From: Artem Konovalov Date: Thu, 18 Jul 2019 18:56:58 +0300 Subject: [PATCH 46/64] fix name of a parameter --- docs/en/query_language/functions/ext_dict_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/query_language/functions/ext_dict_functions.md b/docs/en/query_language/functions/ext_dict_functions.md index 017d941b9f6..959c09f08b0 100644 --- a/docs/en/query_language/functions/ext_dict_functions.md +++ b/docs/en/query_language/functions/ext_dict_functions.md @@ -116,7 +116,7 @@ Type: `UInt8`. For the hierarchical dictionary, returns an array of dictionary keys starting from passed `id_expr` and continuing along the chain of parent elements. ``` -dictGetHierarchy('dict_name', id) +dictGetHierarchy('dict_name', id_expr) ``` **Parameters** From 7d4ebe576d70edf90bd3ef7faddef769d522b3aa Mon Sep 17 00:00:00 2001 From: Artem Konovalov Date: Thu, 18 Jul 2019 18:59:22 +0300 Subject: [PATCH 47/64] fix name of a parameter --- docs/en/query_language/functions/ext_dict_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/query_language/functions/ext_dict_functions.md b/docs/en/query_language/functions/ext_dict_functions.md index 959c09f08b0..6494a2b643e 100644 --- a/docs/en/query_language/functions/ext_dict_functions.md +++ b/docs/en/query_language/functions/ext_dict_functions.md @@ -96,7 +96,7 @@ LIMIT 3 Checks whether the dictionary has the key. ``` -dictHas('dict_name', id) +dictHas('dict_name', id_expr) ``` **Parameters** From 9d5693bd69e7a40d0af9e94edafafe436a30be65 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 19 Jul 2019 02:16:36 +0800 Subject: [PATCH 48/64] fix clang build with certain toolchain --- dbms/src/Common/TaskStatsInfoGetter.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dbms/src/Common/TaskStatsInfoGetter.cpp b/dbms/src/Common/TaskStatsInfoGetter.cpp index 35a68c5a90c..c8a0d5dcbf5 100644 --- a/dbms/src/Common/TaskStatsInfoGetter.cpp +++ b/dbms/src/Common/TaskStatsInfoGetter.cpp @@ -34,6 +34,11 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +// Replace NLMSG_OK with explicit casts since that system macro contains signedness bugs which are not going to be fixed. +static inline bool is_nlmsg_ok(const struct nlmsghdr * const nlh, const ssize_t len) +{ + return len >= static_cast(sizeof(*nlh)) && nlh->nlmsg_len >= sizeof(*nlh) && static_cast(len) >= nlh->nlmsg_len; +} namespace { @@ -128,7 +133,7 @@ struct NetlinkMessage if (header.nlmsg_type == NLMSG_ERROR) throw Exception("Can't receive Netlink response: error " + std::to_string(error.error), ErrorCodes::NETLINK_ERROR); - if (!NLMSG_OK((&header), bytes_received)) + if (!is_nlmsg_ok((&header), bytes_received)) throw Exception("Can't receive Netlink response: wrong number of bytes received", ErrorCodes::NETLINK_ERROR); } }; From d3449e118f63b7cfdb5678827cdbb4c7d08c9f53 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 18 Jul 2019 21:24:38 +0300 Subject: [PATCH 49/64] Update TaskStatsInfoGetter.cpp --- dbms/src/Common/TaskStatsInfoGetter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Common/TaskStatsInfoGetter.cpp b/dbms/src/Common/TaskStatsInfoGetter.cpp index c8a0d5dcbf5..b361161483a 100644 --- a/dbms/src/Common/TaskStatsInfoGetter.cpp +++ b/dbms/src/Common/TaskStatsInfoGetter.cpp @@ -133,7 +133,7 @@ struct NetlinkMessage if (header.nlmsg_type == NLMSG_ERROR) throw Exception("Can't receive Netlink response: error " + std::to_string(error.error), ErrorCodes::NETLINK_ERROR); - if (!is_nlmsg_ok((&header), bytes_received)) + if (!is_nlmsg_ok(&header, bytes_received)) throw Exception("Can't receive Netlink response: wrong number of bytes received", ErrorCodes::NETLINK_ERROR); } }; From 71eed6507e10900141b09af2303f8ca98c646beb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 18 Jul 2019 23:10:31 +0300 Subject: [PATCH 50/64] Removed manual memory tracking when appropriate --- dbms/src/AggregateFunctions/QuantileTiming.h | 7 ------- dbms/src/Common/CombinedCardinalityEstimator.h | 7 ------- dbms/src/Common/HyperLogLogWithSmallSetOptimization.h | 7 ------- 3 files changed, 21 deletions(-) diff --git a/dbms/src/AggregateFunctions/QuantileTiming.h b/dbms/src/AggregateFunctions/QuantileTiming.h index 131ca91dbbf..fbf4da725c0 100644 --- a/dbms/src/AggregateFunctions/QuantileTiming.h +++ b/dbms/src/AggregateFunctions/QuantileTiming.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include #include @@ -513,8 +512,6 @@ private: void mediumToLarge() { - CurrentMemoryTracker::alloc(sizeof(detail::QuantileTimingLarge)); - /// While the data is copied from medium, it is not possible to set `large` value (otherwise it will overwrite some data). detail::QuantileTimingLarge * tmp_large = new detail::QuantileTimingLarge; @@ -528,8 +525,6 @@ private: void tinyToLarge() { - CurrentMemoryTracker::alloc(sizeof(detail::QuantileTimingLarge)); - /// While the data is copied from `medium` it is not possible to set `large` value (otherwise it will overwrite some data). detail::QuantileTimingLarge * tmp_large = new detail::QuantileTimingLarge; @@ -562,8 +557,6 @@ public: else if (kind == Kind::Large) { delete large; - - CurrentMemoryTracker::free(sizeof(detail::QuantileTimingLarge)); } } diff --git a/dbms/src/Common/CombinedCardinalityEstimator.h b/dbms/src/Common/CombinedCardinalityEstimator.h index 824f0a8c018..e048e47cab5 100644 --- a/dbms/src/Common/CombinedCardinalityEstimator.h +++ b/dbms/src/Common/CombinedCardinalityEstimator.h @@ -3,7 +3,6 @@ #include #include #include -#include #include @@ -230,7 +229,6 @@ private: if (getContainerType() != details::ContainerType::SMALL) throw Poco::Exception("Internal error", ErrorCodes::LOGICAL_ERROR); - CurrentMemoryTracker::alloc(sizeof(Medium)); auto tmp_medium = std::make_unique(); for (const auto & x : small) @@ -247,7 +245,6 @@ private: if ((container_type != details::ContainerType::SMALL) && (container_type != details::ContainerType::MEDIUM)) throw Poco::Exception("Internal error", ErrorCodes::LOGICAL_ERROR); - CurrentMemoryTracker::alloc(sizeof(Large)); auto tmp_large = std::make_unique(); if (container_type == details::ContainerType::SMALL) @@ -277,15 +274,11 @@ private: { delete medium; medium = nullptr; - - CurrentMemoryTracker::free(sizeof(Medium)); } else if (container_type == details::ContainerType::LARGE) { delete large; large = nullptr; - - CurrentMemoryTracker::free(sizeof(Large)); } } diff --git a/dbms/src/Common/HyperLogLogWithSmallSetOptimization.h b/dbms/src/Common/HyperLogLogWithSmallSetOptimization.h index 836fbda222e..548b745cb6e 100644 --- a/dbms/src/Common/HyperLogLogWithSmallSetOptimization.h +++ b/dbms/src/Common/HyperLogLogWithSmallSetOptimization.h @@ -4,7 +4,6 @@ #include #include -#include namespace DB @@ -39,8 +38,6 @@ private: void toLarge() { - CurrentMemoryTracker::alloc(sizeof(Large)); - /// At the time of copying data from `tiny`, setting the value of `large` is still not possible (otherwise it will overwrite some data). Large * tmp_large = new Large; @@ -56,11 +53,7 @@ public: ~HyperLogLogWithSmallSetOptimization() { if (isLarge()) - { delete large; - - CurrentMemoryTracker::free(sizeof(Large)); - } } void insert(Key value) From 839acc948ec623fc1cb2dcf23dba86d8a81f992e Mon Sep 17 00:00:00 2001 From: Artem Konovalov Date: Thu, 18 Jul 2019 23:44:05 +0300 Subject: [PATCH 51/64] add russian documentation about functions for working with external dictionaries --- .../functions/ext_dict_functions.md | 200 +++++++++++++++--- 1 file changed, 176 insertions(+), 24 deletions(-) diff --git a/docs/ru/query_language/functions/ext_dict_functions.md b/docs/ru/query_language/functions/ext_dict_functions.md index 8901292aeb2..56aa65976b5 100644 --- a/docs/ru/query_language/functions/ext_dict_functions.md +++ b/docs/ru/query_language/functions/ext_dict_functions.md @@ -1,40 +1,192 @@ # Функции для работы с внешними словарями {#ext_dict_functions} -Информация о подключении и настройке внешних словарей смотрите в разделе [Внешние словари](../dicts/external_dicts.md). +Для получения информации по подключению и настройке внешних словарей, читайте [Внешние словари](../dicts/external_dicts.md). -## dictGetUInt8, dictGetUInt16, dictGetUInt32, dictGetUInt64 +## dictGet -## dictGetInt8, dictGetInt16, dictGetInt32, dictGetInt64 +Получение значения из внешнего словаря -## dictGetFloat32, dictGetFloat64 +``` +dictGet('dict_name', 'attr_name', id_expr) +dictGetOrDefault('dict_name', 'attr_name', id_expr, default_value_expr) +``` -## dictGetDate, dictGetDateTime +**Параметры** -## dictGetUUID +- `dict_name` — Название словаря. [String literal](../syntax.md#syntax-string-literal). +- `attr_name` — Название колонки словаря. [String literal](../syntax.md#syntax-string-literal). +- `id_expr` — Значение ключа. [Expression](../syntax.md#syntax-expressions) Возвращает значение типа [UInt64](../../data_types/int_uint.md) или [Tuple](../../data_types/tuple.md) в зависимости от конфигурации словаря. +- `default_value_expr` — Значение которое возвращается, если словарь не содержит колонку с ключом `id_expr`. [Expression](../syntax.md#syntax-expressions) возвращает значение такого же типа, что и у атрибута `attr_name`. -## dictGetString -`dictGetT('dict_name', 'attr_name', id)` -- получить из словаря dict_name значение атрибута attr_name по ключу id. -`dict_name` и `attr_name` - константные строки. -`id` должен иметь тип UInt64. -Если ключа `id` нет в словаре - вернуть значение по умолчанию, заданное в описании словаря. +**Возвращаемое значение** -## dictGetTOrDefault +- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным типом [attribute's data Тип](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), функция возвращает значение атрибута словаря для заданного `id_expr`. +- Если запращиваемого `id_expr` не оказалось в словаре: -`dictGetT('dict_name', 'attr_name', id, default)` + - `dictGet` возвратит содержимое элемента `` определенного в настройках словаря. + - `dictGetOrDefault` вернет значение переданного `default_value_expr` параметра. -Аналогично функциям `dictGetT`, но значение по умолчанию берётся из последнего аргумента функции. +ClickHouse бросает исключение, если не может обработать значение атрибута или значение не сопоставимо с типом атрибута. -## dictIsIn -`dictIsIn('dict_name', child_id, ancestor_id)` -- для иерархического словаря dict_name - узнать, находится ли ключ child_id внутри ancestor_id (или совпадает с ancestor_id). Возвращает UInt8. +**Примеры использования** + +Создайте файл `ext-dict-text.csv` со следующим содержимым: + +```text +1,1 +2,2 +``` + +Первая колонка - это `id`, вторая - `c1` + +Конфигурация внешнего словаря: + +```xml + + + ext-dict-test + + + /path-to/ext-dict-test.csv + CSV + + + + + + + + id + + + c1 + <Тип>UInt32 + + + + 0 + + +``` + +Выполните запрос: + +```sql +SELECT + dictGetOrDefault('ext-dict-test', 'c1', number + 1, toUInt32(number * 10)) AS val, + toТипName(val) AS Тип +FROM system.numbers +LIMIT 3 +``` +```text +┌─val─┬─Тип───┐ +│ 1 │ UInt32 │ +│ 2 │ UInt32 │ +│ 20 │ UInt32 │ +└─────┴────────┘ +``` + +**Читайте так же** + +- [External Dictionaries](../dicts/external_dicts.md) -## dictGetHierarchy -`dictGetHierarchy('dict_name', id)` -- для иерархического словаря dict_name - вернуть массив ключей словаря, начиная с id и продолжая цепочкой родительских элементов. Возвращает Array(UInt64). ## dictHas -`dictHas('dict_name', id)` -- проверить наличие ключа в словаре. Возвращает значение типа UInt8, равное 0, если ключа нет и 1, если ключ есть. -[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/ext_dict_functions/) +Проверяет наличие строки с заданным ключом в словаре. + +``` +dictHas('dict_name', id_expr) +``` + +**Параметры** + +- `dict_name` — Название словаря. [String literal](../syntax.md#syntax-string-literal). +- `id_expr` — Значение ключа. [Expression](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). + +**Возвращаемое значение** + +- 0, если ключ не был обнаружен +- 1, если ключ присутствует в словаре + +Тип: `UInt8`. + +## dictGetHierarchy + +Для иерархических словарей, возвращает массив ключей, содержащий ключ `id_expr` и все ключи родительских элементов по цепочке. + +``` +dictGetHierarchy('dict_name', id_expr) +``` + +**Параметры** + +- `dict_name` — Название словаря. [String literal](../syntax.md#syntax-string-literal). +- `id_expr` — Значение ключа. [Expression](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). + +**Возвращаемое значение** + +Иерархию ключей словаря. + +Тип: Array(UInt64). + +## dictIsIn + +Осуществляет проверку - является ли ключ родительским в иерархии словаря. + +`dictIsIn ('dict_name', child_id_expr, ancestor_id_expr)` + +**Параметры** + +- `dict_name` — Название словаря. [String literal](../syntax.md#syntax-string-literal). +- `child_id_expr` — Ключ который должен быть проверен. [Expression](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). +- `ancestor_id_expr` — Родительский ключ для ключа `child_id_expr`. [Expression](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). + +**Возвращаемое значение** + +- 0, если `child_id_expr` не является потомком для `ancestor_id_expr`. +- 1, если `child_id_expr` является потомком для `ancestor_id_expr` или если `child_id_expr` равен `ancestor_id_expr`. + +Тип: `UInt8`. + +## Другие функции {#ext_dict_functions-other} + +ClickHouse поддерживает специализированные функции для конвертации значений атрибутов словаря к определенному типу, независимо от настроек словаря. + +Функции: + +- `dictGetInt8`, `dictGetInt16`, `dictGetInt32`, `dictGetInt64` +- `dictGetUInt8`, `dictGetUInt16`, `dictGetUInt32`, `dictGetUInt64` +- `dictGetFloat32`, `dictGetFloat64` +- `dictGetDate` +- `dictGetDateTime` +- `dictGetUUID` +- `dictGetString` + +Все эти функции имеют так же `OrDefault` версию. Например, `dictGetDateOrDefault`. + +Синтаксис: + +``` +dictGet[Тип]('dict_name', 'attr_name', id_expr) +dictGet[Тип]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) +``` + +**Параметры** + +- `dict_name` — Название словаря. [String literal](../syntax.md#syntax-string-literal). +- `attr_name` — Название колонки словаря. [String literal](../syntax.md#syntax-string-literal). +- `id_expr` — Значение ключа. [Expression](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). +- `default_value_expr` — Значение которое возвращается, если словарь не содержит строку с ключом `id_expr`. [Expression](../syntax.md#syntax-expressions) возвращает значение с таким же типом, что и тип атрибута `attr_name`. + +**Возвращаемое значение** + +- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным типом [attribute's data Тип](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), функция возвращает значение атрибута словаря с заданым ключом `id_expr`. +- Если запращиваемого `id_expr` не оказалось в словаре: + + - `dictGet[Тип]` возвратит содержимое элемента `` определенного в настройках словаря. + - `dictGet[Тип]OrDefault` вернет значение переданного `default_value_expr` параметра. + +ClickHouse бросает исключение, если не может обработать значение атрибута или значение не сопоставимо с типом атрибута + +[Исходная статья](https://clickhouse.yandex/docs/en/query_language/functions/ext_dict_functions/) From dcc0433dd6dcb727c913709139d218c260d843f7 Mon Sep 17 00:00:00 2001 From: Artem Konovalov Date: Thu, 18 Jul 2019 23:59:01 +0300 Subject: [PATCH 52/64] fix typos --- .../functions/ext_dict_functions.md | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/ru/query_language/functions/ext_dict_functions.md b/docs/ru/query_language/functions/ext_dict_functions.md index 56aa65976b5..1574e1f93b1 100644 --- a/docs/ru/query_language/functions/ext_dict_functions.md +++ b/docs/ru/query_language/functions/ext_dict_functions.md @@ -1,6 +1,6 @@ # Функции для работы с внешними словарями {#ext_dict_functions} -Для получения информации по подключению и настройке внешних словарей, читайте [Внешние словари](../dicts/external_dicts.md). +Для получения информации по подключению и настройке внешних словарей, читайте [внешние словари](../dicts/external_dicts.md). ## dictGet @@ -13,20 +13,20 @@ dictGetOrDefault('dict_name', 'attr_name', id_expr, default_value_expr) **Параметры** -- `dict_name` — Название словаря. [String literal](../syntax.md#syntax-string-literal). -- `attr_name` — Название колонки словаря. [String literal](../syntax.md#syntax-string-literal). -- `id_expr` — Значение ключа. [Expression](../syntax.md#syntax-expressions) Возвращает значение типа [UInt64](../../data_types/int_uint.md) или [Tuple](../../data_types/tuple.md) в зависимости от конфигурации словаря. -- `default_value_expr` — Значение которое возвращается, если словарь не содержит колонку с ключом `id_expr`. [Expression](../syntax.md#syntax-expressions) возвращает значение такого же типа, что и у атрибута `attr_name`. +- `dict_name` — Название словаря. [Строковый литерал](../syntax.md#syntax-string-literal). +- `attr_name` — Название колонки словаря. [Строковый литерал](../syntax.md#syntax-string-literal). +- `id_expr` — Значение ключа. [Выражение](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md) или [Tuple](../../data_types/tuple.md) в зависимости от конфигурации словаря. +- `default_value_expr` — Значение которое возвращается, если словарь не содержит колонку с ключом `id_expr`. [Выражение](../syntax.md#syntax-expressions) возвращает значение такого же типа, что и у атрибута `attr_name`. **Возвращаемое значение** -- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным типом [attribute's data Тип](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), функция возвращает значение атрибута словаря для заданного `id_expr`. +- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным типом [типы данных атрибутов](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), функция возвращает значение атрибута словаря для заданного `id_expr`. - Если запращиваемого `id_expr` не оказалось в словаре: - `dictGet` возвратит содержимое элемента `` определенного в настройках словаря. - `dictGetOrDefault` вернет значение переданного `default_value_expr` параметра. -ClickHouse бросает исключение, если не может обработать значение атрибута или значение не сопоставимо с типом атрибута. +ClickHouse бросает исключение, если не может обработать значение атрибута или значение несопоставимо с типом атрибута. **Примеры использования** @@ -88,7 +88,7 @@ LIMIT 3 **Читайте так же** -- [External Dictionaries](../dicts/external_dicts.md) +- [Внешние словари](../dicts/external_dicts.md) ## dictHas @@ -101,8 +101,8 @@ dictHas('dict_name', id_expr) **Параметры** -- `dict_name` — Название словаря. [String literal](../syntax.md#syntax-string-literal). -- `id_expr` — Значение ключа. [Expression](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). +- `dict_name` — Название словаря. [Строковый литерал](../syntax.md#syntax-string-literal). +- `id_expr` — Значение ключа. [Выражение](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). **Возвращаемое значение** @@ -121,8 +121,8 @@ dictGetHierarchy('dict_name', id_expr) **Параметры** -- `dict_name` — Название словаря. [String literal](../syntax.md#syntax-string-literal). -- `id_expr` — Значение ключа. [Expression](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). +- `dict_name` — Название словаря. [Строковый литерал](../syntax.md#syntax-string-literal). +- `id_expr` — Значение ключа. [Выражение](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). **Возвращаемое значение** @@ -138,9 +138,9 @@ dictGetHierarchy('dict_name', id_expr) **Параметры** -- `dict_name` — Название словаря. [String literal](../syntax.md#syntax-string-literal). -- `child_id_expr` — Ключ который должен быть проверен. [Expression](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). -- `ancestor_id_expr` — Родительский ключ для ключа `child_id_expr`. [Expression](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). +- `dict_name` — Название словаря. [Строковый литерал](../syntax.md#syntax-string-literal). +- `child_id_expr` — Ключ который должен быть проверен. [Выражение](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). +- `ancestor_id_expr` — Родительский ключ для ключа `child_id_expr`. [Выражение](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). **Возвращаемое значение** @@ -174,19 +174,19 @@ dictGet[Тип]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) **Параметры** -- `dict_name` — Название словаря. [String literal](../syntax.md#syntax-string-literal). -- `attr_name` — Название колонки словаря. [String literal](../syntax.md#syntax-string-literal). -- `id_expr` — Значение ключа. [Expression](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). -- `default_value_expr` — Значение которое возвращается, если словарь не содержит строку с ключом `id_expr`. [Expression](../syntax.md#syntax-expressions) возвращает значение с таким же типом, что и тип атрибута `attr_name`. +- `dict_name` — Название словаря. [Строковый литерал](../syntax.md#syntax-string-literal). +- `attr_name` — Название колонки словаря. [Строковый литерал](../syntax.md#syntax-string-literal). +- `id_expr` — Значение ключа. [Выражение](../syntax.md#syntax-expressions) возвращает значение типа [UInt64](../../data_types/int_uint.md). +- `default_value_expr` — Значение которое возвращается, если словарь не содержит строку с ключом `id_expr`. [Выражение](../syntax.md#syntax-expressions) возвращает значение с таким же типом, что и тип атрибута `attr_name`. **Возвращаемое значение** -- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным типом [attribute's data Тип](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), функция возвращает значение атрибута словаря с заданым ключом `id_expr`. +- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным типом [типы данных атрибута](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), функция возвращает значение атрибута словаря с заданым ключом `id_expr`. - Если запращиваемого `id_expr` не оказалось в словаре: - `dictGet[Тип]` возвратит содержимое элемента `` определенного в настройках словаря. - `dictGet[Тип]OrDefault` вернет значение переданного `default_value_expr` параметра. -ClickHouse бросает исключение, если не может обработать значение атрибута или значение не сопоставимо с типом атрибута +ClickHouse бросает исключение, если не может обработать значение атрибута или значение несопоставимо с типом атрибута [Исходная статья](https://clickhouse.yandex/docs/en/query_language/functions/ext_dict_functions/) From a5196861ffc39bcc7228061e403f79dadc64ec4a Mon Sep 17 00:00:00 2001 From: Artem Konovalov Date: Fri, 19 Jul 2019 00:05:38 +0300 Subject: [PATCH 53/64] fix typos --- docs/ru/query_language/functions/ext_dict_functions.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/ru/query_language/functions/ext_dict_functions.md b/docs/ru/query_language/functions/ext_dict_functions.md index 1574e1f93b1..b1be0b7b906 100644 --- a/docs/ru/query_language/functions/ext_dict_functions.md +++ b/docs/ru/query_language/functions/ext_dict_functions.md @@ -1,6 +1,6 @@ # Функции для работы с внешними словарями {#ext_dict_functions} -Для получения информации по подключению и настройке внешних словарей, читайте [внешние словари](../dicts/external_dicts.md). +Для получения информации по подключению и настройке внешних словарей, читайте раздел про [внешние словари](../dicts/external_dicts.md). ## dictGet @@ -20,7 +20,7 @@ dictGetOrDefault('dict_name', 'attr_name', id_expr, default_value_expr) **Возвращаемое значение** -- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным типом [типы данных атрибутов](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), функция возвращает значение атрибута словаря для заданного `id_expr`. +- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным [типом данных атрибута](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), функция возвращает значение атрибута словаря для заданного `id_expr`. - Если запращиваемого `id_expr` не оказалось в словаре: - `dictGet` возвратит содержимое элемента `` определенного в настройках словаря. @@ -86,7 +86,7 @@ LIMIT 3 └─────┴────────┘ ``` -**Читайте так же** +**Смотрите также** - [Внешние словари](../dicts/external_dicts.md) @@ -181,7 +181,7 @@ dictGet[Тип]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) **Возвращаемое значение** -- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным типом [типы данных атрибута](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), функция возвращает значение атрибута словаря с заданым ключом `id_expr`. +- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным [типом данных атрибута](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), функция возвращает значение атрибута словаря с заданым ключом `id_expr`. - Если запращиваемого `id_expr` не оказалось в словаре: - `dictGet[Тип]` возвратит содержимое элемента `` определенного в настройках словаря. From 1cb5b77fa870fadb34c9bbade950983e1a7ec4c9 Mon Sep 17 00:00:00 2001 From: Artem Konovalov Date: Fri, 19 Jul 2019 00:09:56 +0300 Subject: [PATCH 54/64] fix typos --- docs/ru/query_language/functions/ext_dict_functions.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/query_language/functions/ext_dict_functions.md b/docs/ru/query_language/functions/ext_dict_functions.md index b1be0b7b906..27fa0a31133 100644 --- a/docs/ru/query_language/functions/ext_dict_functions.md +++ b/docs/ru/query_language/functions/ext_dict_functions.md @@ -20,7 +20,7 @@ dictGetOrDefault('dict_name', 'attr_name', id_expr, default_value_expr) **Возвращаемое значение** -- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным [типом данных атрибута](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), функция возвращает значение атрибута словаря для заданного `id_expr`. +- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным [типом данных](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), то функция возвращает значение для заданного ключа `id_expr`. - Если запращиваемого `id_expr` не оказалось в словаре: - `dictGet` возвратит содержимое элемента `` определенного в настройках словаря. @@ -28,7 +28,7 @@ dictGetOrDefault('dict_name', 'attr_name', id_expr, default_value_expr) ClickHouse бросает исключение, если не может обработать значение атрибута или значение несопоставимо с типом атрибута. -**Примеры использования** +**Пример использования** Создайте файл `ext-dict-text.csv` со следующим содержимым: @@ -181,7 +181,7 @@ dictGet[Тип]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) **Возвращаемое значение** -- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным [типом данных атрибута](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), функция возвращает значение атрибута словаря с заданым ключом `id_expr`. +- Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным [типом данных](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes),то функция возвращает значение для заданного ключа `id_expr`. - Если запращиваемого `id_expr` не оказалось в словаре: - `dictGet[Тип]` возвратит содержимое элемента `` определенного в настройках словаря. From 60f3e4dd8c122dbf7d046d61f223f18dbe1e75b8 Mon Sep 17 00:00:00 2001 From: Artem Konovalov Date: Fri, 19 Jul 2019 00:11:00 +0300 Subject: [PATCH 55/64] fix typos --- docs/ru/query_language/functions/ext_dict_functions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/query_language/functions/ext_dict_functions.md b/docs/ru/query_language/functions/ext_dict_functions.md index 27fa0a31133..38e974e2043 100644 --- a/docs/ru/query_language/functions/ext_dict_functions.md +++ b/docs/ru/query_language/functions/ext_dict_functions.md @@ -1,10 +1,10 @@ # Функции для работы с внешними словарями {#ext_dict_functions} -Для получения информации по подключению и настройке внешних словарей, читайте раздел про [внешние словари](../dicts/external_dicts.md). +Для получения информации о подключении и настройке, читайте раздел про [внешние словари](../dicts/external_dicts.md). ## dictGet -Получение значения из внешнего словаря +Получение значения из внешнего словаря. ``` dictGet('dict_name', 'attr_name', id_expr) From 43c13d8283b10b06955b22ef23dfe7c6d6b00bf8 Mon Sep 17 00:00:00 2001 From: Artem Konovalov Date: Fri, 19 Jul 2019 00:15:34 +0300 Subject: [PATCH 56/64] fix typos --- docs/ru/query_language/functions/ext_dict_functions.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/query_language/functions/ext_dict_functions.md b/docs/ru/query_language/functions/ext_dict_functions.md index 38e974e2043..102b72135f9 100644 --- a/docs/ru/query_language/functions/ext_dict_functions.md +++ b/docs/ru/query_language/functions/ext_dict_functions.md @@ -21,7 +21,7 @@ dictGetOrDefault('dict_name', 'attr_name', id_expr, default_value_expr) **Возвращаемое значение** - Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным [типом данных](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes), то функция возвращает значение для заданного ключа `id_expr`. -- Если запращиваемого `id_expr` не оказалось в словаре: +- Если запрашиваемого `id_expr` не оказалось в словаре: - `dictGet` возвратит содержимое элемента `` определенного в настройках словаря. - `dictGetOrDefault` вернет значение переданного `default_value_expr` параметра. @@ -128,7 +128,7 @@ dictGetHierarchy('dict_name', id_expr) Иерархию ключей словаря. -Тип: Array(UInt64). +Тип: [Array(UInt64)](../../data_types/array.md). ## dictIsIn From 8e93b500fdf929053324f75081ac0d0d02a50fed Mon Sep 17 00:00:00 2001 From: Artem Konovalov Date: Fri, 19 Jul 2019 00:16:45 +0300 Subject: [PATCH 57/64] fix typos --- docs/ru/query_language/functions/ext_dict_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/query_language/functions/ext_dict_functions.md b/docs/ru/query_language/functions/ext_dict_functions.md index 102b72135f9..d9a55c112f3 100644 --- a/docs/ru/query_language/functions/ext_dict_functions.md +++ b/docs/ru/query_language/functions/ext_dict_functions.md @@ -189,4 +189,4 @@ dictGet[Тип]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) ClickHouse бросает исключение, если не может обработать значение атрибута или значение несопоставимо с типом атрибута -[Исходная статья](https://clickhouse.yandex/docs/en/query_language/functions/ext_dict_functions/) +[Оригинальная статья](https://clickhouse.yandex/docs/en/query_language/functions/ext_dict_functions/) From c2fee7345d3efe8e6200c1ff022b57a68db5943e Mon Sep 17 00:00:00 2001 From: Artem Konovalov Date: Fri, 19 Jul 2019 00:21:16 +0300 Subject: [PATCH 58/64] fix typos --- docs/ru/query_language/functions/ext_dict_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/query_language/functions/ext_dict_functions.md b/docs/ru/query_language/functions/ext_dict_functions.md index d9a55c112f3..a8d54ce2585 100644 --- a/docs/ru/query_language/functions/ext_dict_functions.md +++ b/docs/ru/query_language/functions/ext_dict_functions.md @@ -182,7 +182,7 @@ dictGet[Тип]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) **Возвращаемое значение** - Если ClickHouse успешно обрабатывает атрибут в соотвествии с указаным [типом данных](../dicts/external_dicts_dict_structure.md#ext_dict_structure-attributes),то функция возвращает значение для заданного ключа `id_expr`. -- Если запращиваемого `id_expr` не оказалось в словаре: +- Если запрашиваемого `id_expr` не оказалось в словаре: - `dictGet[Тип]` возвратит содержимое элемента `` определенного в настройках словаря. - `dictGet[Тип]OrDefault` вернет значение переданного `default_value_expr` параметра. From 7808f944e97eafab93e67a75685014549ecbd7a9 Mon Sep 17 00:00:00 2001 From: Artem Konovalov Date: Fri, 19 Jul 2019 00:36:25 +0300 Subject: [PATCH 59/64] changes after code review --- .../query_language/functions/ext_dict_functions.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/ru/query_language/functions/ext_dict_functions.md b/docs/ru/query_language/functions/ext_dict_functions.md index a8d54ce2585..4ed10f98268 100644 --- a/docs/ru/query_language/functions/ext_dict_functions.md +++ b/docs/ru/query_language/functions/ext_dict_functions.md @@ -60,7 +60,7 @@ ClickHouse бросает исключение, если не может обр c1 - <Тип>UInt32 + UInt32 @@ -74,12 +74,12 @@ ClickHouse бросает исключение, если не может обр ```sql SELECT dictGetOrDefault('ext-dict-test', 'c1', number + 1, toUInt32(number * 10)) AS val, - toТипName(val) AS Тип + toТипName(val) AS Type FROM system.numbers LIMIT 3 ``` ```text -┌─val─┬─Тип───┐ +┌─val─┬─type───┐ │ 1 │ UInt32 │ │ 2 │ UInt32 │ │ 20 │ UInt32 │ @@ -93,7 +93,7 @@ LIMIT 3 ## dictHas -Проверяет наличие строки с заданным ключом в словаре. +Проверяет наличие записи с заданным ключом в словаре. ``` dictHas('dict_name', id_expr) @@ -113,7 +113,7 @@ dictHas('dict_name', id_expr) ## dictGetHierarchy -Для иерархических словарей, возвращает массив ключей, содержащий ключ `id_expr` и все ключи родительских элементов по цепочке. +Для иерархических словарей возвращает массив ключей, содержащий ключ `id_expr` и все ключи родительских элементов по цепочке. ``` dictGetHierarchy('dict_name', id_expr) @@ -189,4 +189,4 @@ dictGet[Тип]OrDefault('dict_name', 'attr_name', id_expr, default_value_expr) ClickHouse бросает исключение, если не может обработать значение атрибута или значение несопоставимо с типом атрибута -[Оригинальная статья](https://clickhouse.yandex/docs/en/query_language/functions/ext_dict_functions/) +[Оригинальная статья](https://clickhouse.yandex/docs/ru/query_language/functions/ext_dict_functions/) From 0a3bd5ed00132e23b0111ad20c502d76fb09f38f Mon Sep 17 00:00:00 2001 From: Artem Konovalov Date: Fri, 19 Jul 2019 00:41:06 +0300 Subject: [PATCH 60/64] changes after code review --- docs/ru/query_language/functions/ext_dict_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/query_language/functions/ext_dict_functions.md b/docs/ru/query_language/functions/ext_dict_functions.md index 4ed10f98268..6e0c1172934 100644 --- a/docs/ru/query_language/functions/ext_dict_functions.md +++ b/docs/ru/query_language/functions/ext_dict_functions.md @@ -132,7 +132,7 @@ dictGetHierarchy('dict_name', id_expr) ## dictIsIn -Осуществляет проверку - является ли ключ родительским в иерархии словаря. +Осуществляет проверку - является ли ключ родительским во всех цепочке иерархии словаря. `dictIsIn ('dict_name', child_id_expr, ancestor_id_expr)` From 86d421cee6454276b2f9b9afa9b544efc475013d Mon Sep 17 00:00:00 2001 From: Artem Konovalov Date: Fri, 19 Jul 2019 00:42:48 +0300 Subject: [PATCH 61/64] changes after code review --- docs/ru/query_language/functions/ext_dict_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/query_language/functions/ext_dict_functions.md b/docs/ru/query_language/functions/ext_dict_functions.md index 6e0c1172934..3fb4a110e88 100644 --- a/docs/ru/query_language/functions/ext_dict_functions.md +++ b/docs/ru/query_language/functions/ext_dict_functions.md @@ -132,7 +132,7 @@ dictGetHierarchy('dict_name', id_expr) ## dictIsIn -Осуществляет проверку - является ли ключ родительским во всех цепочке иерархии словаря. +Осуществляет проверку - является ли ключ родительским во всей иерархической цепочке словаря. `dictIsIn ('dict_name', child_id_expr, ancestor_id_expr)` From 9fb3135c23e27032d99b64eda2604fb0b342a7c2 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 19 Jul 2019 03:39:22 +0300 Subject: [PATCH 62/64] Improve utility zookeeper-adjust-block-numbers-to-parts. Add filter on shards and tables. Add flag --dry-run. The utility now checks cversion before doing anything. --- utils/CMakeLists.txt | 1 + .../main.cpp | 203 ++++++++++++++---- 2 files changed, 158 insertions(+), 46 deletions(-) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 3b523822451..b3df25d13e6 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -24,6 +24,7 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS) add_subdirectory (zookeeper-copy-tree) add_subdirectory (zookeeper-remove-by-list) add_subdirectory (zookeeper-create-entry-to-download-part) + add_subdirectory (zookeeper-adjust-block-numbers-to-parts) add_subdirectory (wikistat-loader) add_subdirectory (fill-factor) add_subdirectory (check-marks) diff --git a/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp b/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp index dda1677f3a4..099a9e64153 100644 --- a/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp +++ b/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp @@ -1,13 +1,55 @@ #include #include #include +#include #include #include #include #include -size_t getMaxBlockSizeForPartition(zkutil::ZooKeeper & zk, + +std::vector getAllShards(zkutil::ZooKeeper & zk, const std::string & root) +{ + return zk.getChildren(root); +} + + +std::vector removeNotExistingShards(zkutil::ZooKeeper & zk, const std::string & root, const std::vector & shards) +{ + auto existing_shards = getAllShards(zk, root); + std::vector filtered_shards; + filtered_shards.reserve(shards.size()); + for (const auto & shard : shards) + if (std::find(existing_shards.begin(), existing_shards.end(), shard) == existing_shards.end()) + std::cerr << "Shard " << shard << " not found." << std::endl; + else + filtered_shards.emplace_back(shard); + return filtered_shards; +} + + +std::vector getAllTables(zkutil::ZooKeeper & zk, const std::string & root, const std::string & shard) +{ + return zk.getChildren(root + "/" + shard); +} + + +std::vector removeNotExistingTables(zkutil::ZooKeeper & zk, const std::string & root, const std::string & shard, const std::vector & tables) +{ + auto existing_tables = getAllTables(zk, root, shard); + std::vector filtered_tables; + filtered_tables.reserve(tables.size()); + for (const auto & table : tables) + if (std::find(existing_tables.begin(), existing_tables.end(), table) == existing_tables.end()) + std::cerr << "\tTable " << table << " not found on shard " << shard << "." << std::endl; + else + filtered_tables.emplace_back(table); + return filtered_tables; +} + + +size_t getMaxBlockNumberForPartition(zkutil::ZooKeeper & zk, const std::string & replica_path, const std::string & partition_name, const DB::MergeTreeDataFormatVersion & format_version) @@ -28,36 +70,74 @@ size_t getMaxBlockSizeForPartition(zkutil::ZooKeeper & zk, } catch (const DB::Exception & ex) { - std::cerr << "Exception on: " << ex.displayText() << " will skip part: " << part << std::endl; + std::cerr << ex.displayText() << ", Part " << part << "skipped." << std::endl; } } } return max_block_num; } -std::unordered_map getAllTablesBlockPaths(zkutil::ZooKeeper & zk, const std::string & root) + +size_t getCurrentBlockNumberForPartition(zkutil::ZooKeeper & zk, const std::string & part_path) +{ + Coordination::Stat stat; + zk.get(part_path, &stat); + + /// References: + /// https://stackoverflow.com/a/10347910 + /// https://bowenli86.github.io/2016/07/07/distributed%20system/zookeeper/How-does-ZooKeeper-s-persistent-sequential-id-work/ + return (stat.cversion + stat.numChildren) / 2; +} + + +std::unordered_map getPartitionsNeedAdjustingBlockNumbers( + zkutil::ZooKeeper & zk, const std::string & root, const std::vector & shards, const std::vector & tables) { std::unordered_map result; - auto shards = zk.getChildren(root); - for (const auto & shard : shards) + + std::vector use_shards = shards.empty() ? getAllShards(zk, root) : removeNotExistingShards(zk, root, shards); + + for (const auto & shard : use_shards) { - std::string shard_path = root + "/" + shard; - auto tables = zk.getChildren(shard_path); - for (auto table : tables) + std::cout << "Shard: " << shard << std::endl; + std::vector use_tables = tables.empty() ? getAllTables(zk, root, shard) : removeNotExistingTables(zk, root, shard, tables); + + for (auto table : use_tables) { - std::cerr << "Searching for nodes in: " << table << std::endl; - std::string table_path = shard_path + "/" + table; - auto format_version = DB::ReplicatedMergeTreeTableMetadata::parse(zk.get(table_path + "/metadata")).data_format_version; + std::cout << "\tTable: " << table << std::endl; + std::string table_path = root + "/" + shard + "/" + table; std::string blocks_path = table_path + "/block_numbers"; - auto partitions = zk.getChildren(blocks_path); - if (!partitions.empty()) + + std::vector partitions; + DB::MergeTreeDataFormatVersion format_version; + try { - for (auto partition : partitions) + format_version = DB::ReplicatedMergeTreeTableMetadata::parse(zk.get(table_path + "/metadata")).data_format_version; + partitions = zk.getChildren(blocks_path); + } + catch (const DB::Exception & ex) + { + std::cerr << ex.displayText() << ", table " << table << " skipped." << std::endl; + continue; + } + + for (auto partition : partitions) + { + try { std::string part_path = blocks_path + "/" + partition; - size_t partition_max_block = getMaxBlockSizeForPartition(zk, table_path, partition, format_version); - std::cerr << "\tFound max block number: " << partition_max_block << " for part: " << partition << std::endl; - result.emplace(part_path, partition_max_block); + size_t partition_max_block = getMaxBlockNumberForPartition(zk, table_path, partition, format_version); + size_t current_block_number = getCurrentBlockNumberForPartition(zk, part_path); + if (current_block_number <= partition_max_block) + { + std::cout << "\t\tPartition: " << partition << ": current block_number: " << current_block_number + << ", max block number: " << partition_max_block << ". Adjusting is required." << std::endl; + result.emplace(part_path, partition_max_block); + } + } + catch (const DB::Exception & ex) + { + std::cerr << ex.displayText() << ", partition " << partition << " skipped." << std::endl; } } } @@ -66,67 +146,98 @@ std::unordered_map getAllTablesBlockPaths(zkutil::ZooKeeper } -void rotateNodes(zkutil::ZooKeeper & zk, const std::string & path, size_t max_block_num) +void setCurrentBlockNumber(zkutil::ZooKeeper & zk, const std::string & path, size_t new_current_block_number) { - Coordination::Requests requests; std::string block_prefix = path + "/block-"; - std::string current = zk.create(block_prefix, "", zkutil::CreateMode::EphemeralSequential); - size_t current_block_num = DB::parse(current.c_str() + block_prefix.size(), current.size() - block_prefix.size()); - if (current_block_num >= max_block_num) - { - std::cerr << "Nothing to rotate, current block num: " << current_block_num << " max_block_num:" << max_block_num << std::endl; - return; - } + Coordination::Requests requests; - size_t need_to_rotate = max_block_num - current_block_num; - std::cerr << "Will rotate: " << need_to_rotate << " block numbers from " << current_block_num << " to " << max_block_num << std::endl; - - for (size_t i = 0; i < need_to_rotate; ++i) + for (size_t current_block_number = getCurrentBlockNumberForPartition(zk, path); + current_block_number < new_current_block_number; + ++current_block_number) { if (requests.size() == 50) { - std::cerr << "Rotating: " << i << " block numbers" << std::endl; zk.multi(requests); + std::cout << path << ": " << requests.size() << " ephemeral sequential nodes inserted." << std::endl; requests.clear(); } requests.emplace_back(zkutil::makeCreateRequest(path + "/block-", "", zkutil::CreateMode::EphemeralSequential)); } + if (!requests.empty()) { + std::cout << path << ": " << requests.size() << " ephemeral sequential nodes inserted." << std::endl; zk.multi(requests); } } + int main(int argc, char ** argv) try { - boost::program_options::options_description desc("Allowed options"); + /// Parse the command line. + namespace po = boost::program_options; + po::options_description desc("Allowed options"); desc.add_options() - ("help,h", "produce help message") - ("address,a", boost::program_options::value()->required(), "addresses of ZooKeeper instances, comma separated. Example: example01e.yandex.ru:2181") - ("path,p", boost::program_options::value()->required(), "path of replica queue to insert node (without trailing slash)"); + ("help,h", "show help") + ("zookeeper,z", po::value(), "Addresses of ZooKeeper instances, comma-separated. Example: example01e.yandex.ru:2181") + ("path,p", po::value(), "[optional] Path of replica queue to insert node (without trailing slash). By default it's /clickhouse/tables") + ("shard,s", po::value(), "[optional] Shards to process, comma-separated. If not specified then the utility will process all the shards.") + ("table,t", po::value(), "[optional] Tables to process, comma-separated. If not specified then the utility will process all the tables.") + ("dry-run", "[optional] Specify if you want this utility just to analyze block numbers without any changes."); - boost::program_options::variables_map options; - boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); + po::variables_map options; + po::store(po::parse_command_line(argc, argv, desc), options); - if (options.count("help")) + auto show_usage = [&] { - std::cout << "Util for /block_numbers node adjust with max block number in partition" << std::endl; - std::cout << "Usage: " << argv[0] << " [options]" << std::endl; + std::cout << "Usage: " << std::endl; + std::cout << " " << argv[0] << " [options]" << std::endl; std::cout << desc << std::endl; + }; + + if (options.count("help") || (argc == 1)) + { + std::cout << "This utility adjusts the /block_numbers zookeeper nodes to the correct block number in partition." << std::endl; + std::cout << "It might be useful when incorrect block numbers stored in zookeeper don't allow you to insert data into a table or drop/detach a partition." << std::endl; + show_usage(); + return 0; + } + + if (!options.count("zookeeper")) + { + std::cerr << "Option --zookeeper should be set." << std::endl; + show_usage(); return 1; } - std::string global_path = options.at("path").as(); + std::string root = options.count("path") ? options.at("path").as() : "/clickhouse/tables"; - zkutil::ZooKeeper zookeeper(options.at("address").as()); + std::vector shards, tables; + if (options.count("shard")) + boost::split(shards, options.at("shard").as(), boost::algorithm::is_any_of(",")); + if (options.count("table")) + boost::split(tables, options.at("table").as(), boost::algorithm::is_any_of(",")); - auto all_path = getAllTablesBlockPaths(zookeeper, global_path); - for (const auto & [path, max_block_num] : all_path) + /// Check if the adjusting of the block numbers is required. + std::cout << "Checking if adjusting of the block numbers is required:" << std::endl; + zkutil::ZooKeeper zookeeper(options.at("zookeeper").as()); + auto part_paths_with_max_block_numbers = getPartitionsNeedAdjustingBlockNumbers(zookeeper, root, shards, tables); + + if (part_paths_with_max_block_numbers.empty()) { - std::cerr << "Rotating on: " << path << std::endl; - rotateNodes(zookeeper, path, max_block_num); + std::cout << "No adjusting required." << std::endl; + return 0; } + + /// Adjust the block numbers. + if (options.count("dry-run")) + return 0; + + std::cout << std::endl << "Adjusting the block numbers:" << std::endl; + for (const auto & [part_path, max_block_number] : part_paths_with_max_block_numbers) + setCurrentBlockNumber(zookeeper, part_path, max_block_number + 1); + return 0; } catch (const Poco::Exception & e) From e6aebb5b135674256d6df51dfc9c062a6ca6e96e Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 19 Jul 2019 12:19:47 +0300 Subject: [PATCH 63/64] Fix after review. --- .../main.cpp | 91 +++++++++++++------ 1 file changed, 65 insertions(+), 26 deletions(-) diff --git a/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp b/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp index 099a9e64153..3e449043adc 100644 --- a/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp +++ b/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp @@ -49,14 +49,14 @@ std::vector removeNotExistingTables(zkutil::ZooKeeper & zk, const s } -size_t getMaxBlockNumberForPartition(zkutil::ZooKeeper & zk, +Int64 getMaxBlockNumberForPartition(zkutil::ZooKeeper & zk, const std::string & replica_path, const std::string & partition_name, const DB::MergeTreeDataFormatVersion & format_version) { auto replicas_path = replica_path + "/replicas"; auto replica_hosts = zk.getChildren(replicas_path); - size_t max_block_num = 0; + Int64 max_block_num = 0; for (const auto & replica_host : replica_hosts) { auto parts = zk.getChildren(replicas_path + "/" + replica_host + "/parts"); @@ -66,7 +66,7 @@ size_t getMaxBlockNumberForPartition(zkutil::ZooKeeper & zk, { auto info = DB::MergeTreePartInfo::fromPartName(part, format_version); if (info.partition_id == partition_name) - max_block_num = std::max(info.max_block, max_block_num); + max_block_num = std::max(info.max_block, max_block_num); } catch (const DB::Exception & ex) { @@ -78,7 +78,7 @@ size_t getMaxBlockNumberForPartition(zkutil::ZooKeeper & zk, } -size_t getCurrentBlockNumberForPartition(zkutil::ZooKeeper & zk, const std::string & part_path) +Int64 getCurrentBlockNumberForPartition(zkutil::ZooKeeper & zk, const std::string & part_path) { Coordination::Stat stat; zk.get(part_path, &stat); @@ -90,10 +90,10 @@ size_t getCurrentBlockNumberForPartition(zkutil::ZooKeeper & zk, const std::stri } -std::unordered_map getPartitionsNeedAdjustingBlockNumbers( +std::unordered_map getPartitionsNeedAdjustingBlockNumbers( zkutil::ZooKeeper & zk, const std::string & root, const std::vector & shards, const std::vector & tables) { - std::unordered_map result; + std::unordered_map result; std::vector use_shards = shards.empty() ? getAllShards(zk, root) : removeNotExistingShards(zk, root, shards); @@ -126,9 +126,9 @@ std::unordered_map getPartitionsNeedAdjustingBlockNumbers( try { std::string part_path = blocks_path + "/" + partition; - size_t partition_max_block = getMaxBlockNumberForPartition(zk, table_path, partition, format_version); - size_t current_block_number = getCurrentBlockNumberForPartition(zk, part_path); - if (current_block_number <= partition_max_block) + Int64 partition_max_block = getMaxBlockNumberForPartition(zk, table_path, partition, format_version); + Int64 current_block_number = getCurrentBlockNumberForPartition(zk, part_path); + if (current_block_number < partition_max_block + 1) { std::cout << "\t\tPartition: " << partition << ": current block_number: " << current_block_number << ", max block number: " << partition_max_block << ". Adjusting is required." << std::endl; @@ -146,29 +146,63 @@ std::unordered_map getPartitionsNeedAdjustingBlockNumbers( } -void setCurrentBlockNumber(zkutil::ZooKeeper & zk, const std::string & path, size_t new_current_block_number) +void setCurrentBlockNumber(zkutil::ZooKeeper & zk, const std::string & path, Int64 new_current_block_number) { - std::string block_prefix = path + "/block-"; - Coordination::Requests requests; + Int64 current_block_number = getCurrentBlockNumberForPartition(zk, path); - for (size_t current_block_number = getCurrentBlockNumberForPartition(zk, path); - current_block_number < new_current_block_number; - ++current_block_number) + auto create_ephemeral_nodes = [&](size_t count) { - if (requests.size() == 50) + std::string block_prefix = path + "/block-"; + Coordination::Requests requests; + requests.reserve(count); + for (size_t i = 0; i != count; ++i) + requests.emplace_back(zkutil::makeCreateRequest(block_prefix, "", zkutil::CreateMode::EphemeralSequential)); + auto responses = zk.multi(requests); + + std::vector paths_created; + paths_created.reserve(responses.size()); + for (const auto & response : responses) { - zk.multi(requests); - std::cout << path << ": " << requests.size() << " ephemeral sequential nodes inserted." << std::endl; - requests.clear(); + const auto * create_response = dynamic_cast(response.get()); + if (!create_response) + { + std::cerr << "\tCould not create ephemeral node " << block_prefix << std::endl; + return false; + } + paths_created.emplace_back(create_response->path_created); } - requests.emplace_back(zkutil::makeCreateRequest(path + "/block-", "", zkutil::CreateMode::EphemeralSequential)); - } - if (!requests.empty()) - { - std::cout << path << ": " << requests.size() << " ephemeral sequential nodes inserted." << std::endl; - zk.multi(requests); - } + std::sort(paths_created.begin(), paths_created.end()); + for (const auto & path_created : paths_created) + { + Int64 number = DB::parse(path_created.c_str() + block_prefix.size(), path_created.size() - block_prefix.size()); + if (number != current_block_number) + { + char suffix[11] = ""; + sprintf(suffix, "%010ld", current_block_number); + std::string expected_path = block_prefix + suffix; + std::cerr << "\t" << path_created << ": Ephemeral node has been created with an unexpected path (expected something like " + << expected_path << ")." << std::endl; + return false; + } + std::cout << "\t" << path_created << std::endl; + ++current_block_number; + } + + return true; + }; + + if (current_block_number >= new_current_block_number) + return; + + std::cout << "Creating ephemeral sequential nodes:" << std::endl; + create_ephemeral_nodes(1); /// Firstly try to create just a single node. + + /// Create other nodes in batches of 50 nodes. + while (current_block_number + 50 <= new_current_block_number) + create_ephemeral_nodes(50); + + create_ephemeral_nodes(new_current_block_number - current_block_number); } @@ -230,9 +264,14 @@ try return 0; } + std::cout << "Required adjusting of " << part_paths_with_max_block_numbers.size() << " block numbers." << std::endl; + /// Adjust the block numbers. if (options.count("dry-run")) + { + std::cout << "This is a dry-run, exiting." << std::endl; return 0; + } std::cout << std::endl << "Adjusting the block numbers:" << std::endl; for (const auto & [part_path, max_block_number] : part_paths_with_max_block_numbers) From 98b0d08bf3407d04bdb047439b8e582555e67cd3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 19 Jul 2019 15:57:23 +0300 Subject: [PATCH 64/64] Added missing header #5981 --- dbms/src/Common/new_delete.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Common/new_delete.cpp b/dbms/src/Common/new_delete.cpp index d9140f9459d..aff708135e1 100644 --- a/dbms/src/Common/new_delete.cpp +++ b/dbms/src/Common/new_delete.cpp @@ -1,5 +1,6 @@ #include +#include #include #include