From f306ccde2c26768409d9ac84b96905bfbe8aac4a Mon Sep 17 00:00:00 2001 From: Hiroaki Nakamura Date: Sun, 27 Aug 2017 10:58:57 +0900 Subject: [PATCH 01/32] Add clang to Build-Depends It is used by debian/copy_clang_binaries.sh --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index a91d321d09d..e1e484beedc 100644 --- a/debian/control +++ b/debian/control @@ -3,6 +3,7 @@ Priority: optional Maintainer: Alexey Milovidov Build-Depends: debhelper (>= 9), cmake, + clang, gcc-6, g++-6, default-libmysqlclient-dev | libmysqlclient-dev, libicu-dev, From 4394b840f45eefb1257d7ab92e2759a2ebba54ed Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 28 Aug 2017 21:00:16 +0300 Subject: [PATCH 02/32] Revert "Add clang to Build-Depends" This reverts commit f306ccde2c26768409d9ac84b96905bfbe8aac4a. --- debian/control | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/control b/debian/control index e1e484beedc..a91d321d09d 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,6 @@ Priority: optional Maintainer: Alexey Milovidov Build-Depends: debhelper (>= 9), cmake, - clang, gcc-6, g++-6, default-libmysqlclient-dev | libmysqlclient-dev, libicu-dev, From 6c26eedcd74a89b1f228033a852c1c3c0f573b5a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 28 Aug 2017 21:33:25 +0300 Subject: [PATCH 03/32] Fixed wrong documentation [#CLICKHOUSE-2]. --- docs/en/table_engines/replication.rst | 2 +- docs/ru/table_engines/replication.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/table_engines/replication.rst b/docs/en/table_engines/replication.rst index 44b86bc1bb1..a3d543f7059 100644 --- a/docs/en/table_engines/replication.rst +++ b/docs/en/table_engines/replication.rst @@ -159,7 +159,7 @@ If the data differs on various replicas, first sync it, or delete this data on a Rename the existing MergeTree table, then create a ReplicatedMergeTree table with the old name. Move the data from the old table to the 'detached' subdirectory inside the directory with the new table data (``/var/lib/clickhouse/data/db_name/table_name/``). -Then run ALTER TABLE ATTACH PART on one of the replicas to add these data parts to the working set. +Then run ALTER TABLE ATTACH PARTITION on one of the replicas to add these data parts to the working set. If exactly the same parts exist on the other replicas, they are added to the working set on them. If not, the parts are downloaded from the replica that has them. diff --git a/docs/ru/table_engines/replication.rst b/docs/ru/table_engines/replication.rst index 83f97928dea..24bc1b619d1 100644 --- a/docs/ru/table_engines/replication.rst +++ b/docs/ru/table_engines/replication.rst @@ -163,7 +163,7 @@ ReplicatedSummingMergeTree Переименуйте имеющуюся MergeTree таблицу, затем создайте со старым именем таблицу типа ReplicatedMergeTree. Перенесите данные из старой таблицы в поддиректорию detached в директории с данными новой таблицы (``/var/lib/clickhouse/data/db_name/table_name/``). -Затем добавьте эти куски данных в рабочий набор с помощью выполнения запросов ALTER TABLE ATTACH PART на одной из реплик. +Затем добавьте эти куски данных в рабочий набор с помощью выполнения запросов ALTER TABLE ATTACH PARTITION на одной из реплик. Преобразование из ReplicatedMergeTree в MergeTree ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 4dc36893d3f74e444ca8655fa9fe3a83ff7745c3 Mon Sep 17 00:00:00 2001 From: Vadim Skipin Date: Thu, 24 Aug 2017 17:51:13 +0300 Subject: [PATCH 04/32] Keep config in context to avoid Application singleton usages --- .../Embedded/RegionsHierarchies.cpp | 19 +++++----- .../Embedded/RegionsHierarchies.h | 9 +++-- .../Embedded/RegionsHierarchy.cpp | 7 ---- .../Dictionaries/Embedded/RegionsHierarchy.h | 12 ------ .../Dictionaries/Embedded/RegionsNames.cpp | 9 ++--- dbms/src/Dictionaries/Embedded/RegionsNames.h | 13 ++++--- .../Embedded/TechDataHierarchy.cpp | 7 ++-- .../Dictionaries/Embedded/TechDataHierarchy.h | 10 +++-- dbms/src/Interpreters/Context.cpp | 37 +++++++++++++++---- dbms/src/Interpreters/Context.h | 6 ++- .../src/Interpreters/EmbeddedDictionaries.cpp | 29 ++++++++------- dbms/src/Interpreters/EmbeddedDictionaries.h | 9 ++--- .../src/Interpreters/ExternalDictionaries.cpp | 3 +- dbms/src/Server/HTTPHandler.cpp | 7 ++-- dbms/src/Server/MetricsTransmitter.cpp | 18 ++++++--- dbms/src/Server/MetricsTransmitter.h | 14 ++++++- dbms/src/Server/Server.cpp | 3 +- dbms/src/Storages/StorageFactory.cpp | 2 +- .../Storages/System/StorageSystemGraphite.cpp | 17 +++++---- 19 files changed, 134 insertions(+), 97 deletions(-) diff --git a/dbms/src/Dictionaries/Embedded/RegionsHierarchies.cpp b/dbms/src/Dictionaries/Embedded/RegionsHierarchies.cpp index e64cf3edf30..1a61f7a0478 100644 --- a/dbms/src/Dictionaries/Embedded/RegionsHierarchies.cpp +++ b/dbms/src/Dictionaries/Embedded/RegionsHierarchies.cpp @@ -1,20 +1,19 @@ #include -#include -#include -#include + #include +#include + static constexpr auto config_key = "path_to_regions_hierarchy_file"; -RegionsHierarchies::RegionsHierarchies() -: RegionsHierarchies(Poco::Util::Application::instance().config().getString(config_key)) +void RegionsHierarchies::reload(const Poco::Util::AbstractConfiguration & config) { + reload(config.getString(config_key)); } - -RegionsHierarchies::RegionsHierarchies(const std::string & path) +void RegionsHierarchies::reload(const std::string & path) { Logger * log = &Logger::get("RegionsHierarchies"); @@ -47,10 +46,12 @@ RegionsHierarchies::RegionsHierarchies(const std::string & path) std::forward_as_tuple(dir_it->path())); } } + + reload(); } -bool RegionsHierarchies::isConfigured() +bool RegionsHierarchies::isConfigured(const Poco::Util::AbstractConfiguration & config) { - return Poco::Util::Application::instance().config().has(config_key); + return config.has(config_key); } diff --git a/dbms/src/Dictionaries/Embedded/RegionsHierarchies.h b/dbms/src/Dictionaries/Embedded/RegionsHierarchies.h index 5b0be5cfb66..7dfc0bfb555 100644 --- a/dbms/src/Dictionaries/Embedded/RegionsHierarchies.h +++ b/dbms/src/Dictionaries/Embedded/RegionsHierarchies.h @@ -1,7 +1,10 @@ #pragma once #include + +#include #include + #include @@ -24,11 +27,11 @@ public: * For example, if /opt/geo/regions_hierarchy.txt is specified, * then the /opt/geo/regions_hierarchy_ua.txt file will also be loaded, if any, it will be accessible by the `ua` key. */ - RegionsHierarchies(); - explicit RegionsHierarchies(const std::string & path_to_regions_hierarchy_file); + void reload(const Poco::Util::AbstractConfiguration & config); + void reload(const std::string & directory); /// Has corresponding section in configuration file. - static bool isConfigured(); + static bool isConfigured(const Poco::Util::AbstractConfiguration & config); /** Reloads, if necessary, all hierarchies of regions. diff --git a/dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp b/dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp index 30e06ede014..928d0e39cd5 100644 --- a/dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp +++ b/dbms/src/Dictionaries/Embedded/RegionsHierarchy.cpp @@ -12,13 +12,6 @@ #include -static constexpr auto config_key = "path_to_regions_hierarchy_file"; - - -RegionsHierarchy::RegionsHierarchy() -{ - path = Poco::Util::Application::instance().config().getString(config_key); -} RegionsHierarchy::RegionsHierarchy(const std::string & path_) { diff --git a/dbms/src/Dictionaries/Embedded/RegionsHierarchy.h b/dbms/src/Dictionaries/Embedded/RegionsHierarchy.h index 0b819e51333..4c3c0905bf7 100644 --- a/dbms/src/Dictionaries/Embedded/RegionsHierarchy.h +++ b/dbms/src/Dictionaries/Embedded/RegionsHierarchy.h @@ -3,7 +3,6 @@ #include #include #include -#include #define REGION_TYPE_CITY 6 @@ -61,7 +60,6 @@ private: std::string path; public: - RegionsHierarchy(); RegionsHierarchy(const std::string & path_); /// Reloads, if necessary, the hierarchy of regions. Not threadsafe. @@ -142,13 +140,3 @@ public: return populations[region]; } }; - - -class RegionsHierarchySingleton : public ext::singleton, public RegionsHierarchy -{ -friend class ext::singleton; -protected: - RegionsHierarchySingleton() - { - } -}; diff --git a/dbms/src/Dictionaries/Embedded/RegionsNames.cpp b/dbms/src/Dictionaries/Embedded/RegionsNames.cpp index ef9f7d6e8d0..ebaebede43e 100644 --- a/dbms/src/Dictionaries/Embedded/RegionsNames.cpp +++ b/dbms/src/Dictionaries/Embedded/RegionsNames.cpp @@ -28,10 +28,9 @@ std::string RegionsNames::dumpSupportedLanguagesNames() return res; } -void RegionsNames::reload() +void RegionsNames::reload(const Poco::Util::AbstractConfiguration & config) { - std::string directory = Poco::Util::Application::instance().config().getString(config_key); - reload(directory); + reload(config.getString(config_key)); } void RegionsNames::reload(const std::string & directory) @@ -110,7 +109,7 @@ void RegionsNames::reload(const std::string & directory) } -bool RegionsNames::isConfigured() +bool RegionsNames::isConfigured(const Poco::Util::AbstractConfiguration & config) { - return Poco::Util::Application::instance().config().has(config_key); + return config.has(config_key); } diff --git a/dbms/src/Dictionaries/Embedded/RegionsNames.h b/dbms/src/Dictionaries/Embedded/RegionsNames.h index c1399cca8ad..52b145d1718 100644 --- a/dbms/src/Dictionaries/Embedded/RegionsNames.h +++ b/dbms/src/Dictionaries/Embedded/RegionsNames.h @@ -1,10 +1,13 @@ #pragma once +#include +#include + +#include +#include + #include #include -#include -#include -#include /** A class that allows you to recognize by region id its text name in one of the supported languages: ru, en, ua, by, kz, tr. @@ -67,11 +70,11 @@ private: public: /** Reboot, if necessary, the names of regions. */ - void reload(); + void reload(const Poco::Util::AbstractConfiguration & config); void reload(const std::string & directory); /// Has corresponding section in configuration file. - static bool isConfigured(); + static bool isConfigured(const Poco::Util::AbstractConfiguration & config); StringRef getRegionName(RegionID region_id, Language language = Language::RU) const diff --git a/dbms/src/Dictionaries/Embedded/TechDataHierarchy.cpp b/dbms/src/Dictionaries/Embedded/TechDataHierarchy.cpp index 07baa4ebf19..921b7f5e9ee 100644 --- a/dbms/src/Dictionaries/Embedded/TechDataHierarchy.cpp +++ b/dbms/src/Dictionaries/Embedded/TechDataHierarchy.cpp @@ -10,10 +10,9 @@ static constexpr auto config_key = "mysql_metrica"; -void TechDataHierarchy::reload() +void TechDataHierarchy::reload(const Poco::Util::AbstractConfiguration & config) { Logger * log = &Logger::get("TechDataHierarchy"); - LOG_DEBUG(log, "Loading tech data hierarchy."); mysqlxx::PoolWithFailover pool(config_key); @@ -53,9 +52,9 @@ void TechDataHierarchy::reload() } -bool TechDataHierarchy::isConfigured() +bool TechDataHierarchy::isConfigured(const Poco::Util::AbstractConfiguration & config) { - return Poco::Util::Application::instance().config().has(config_key); + return config.has(config_key); } #endif diff --git a/dbms/src/Dictionaries/Embedded/TechDataHierarchy.h b/dbms/src/Dictionaries/Embedded/TechDataHierarchy.h index cdfd9f1a526..c86d188aa92 100644 --- a/dbms/src/Dictionaries/Embedded/TechDataHierarchy.h +++ b/dbms/src/Dictionaries/Embedded/TechDataHierarchy.h @@ -1,8 +1,12 @@ #pragma once -#include +#include +#include + #include +#include + /** @brief Class that lets you know if a search engine or operating system belongs * another search engine or operating system, respectively. @@ -15,10 +19,10 @@ private: UInt8 se_parent[256] {}; public: - void reload(); + void reload(const Poco::Util::AbstractConfiguration & config); /// Has corresponding section in configuration file. - static bool isConfigured(); + static bool isConfigured(const Poco::Util::AbstractConfiguration & config); /// The "belongs" relation. diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index c011734bdb1..42a4b1bcba0 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -104,6 +104,8 @@ struct ContextShared String path; /// Path to the data directory, with a slash at the end. String tmp_path; /// The path to the temporary files that occur when processing the request. String flags_path; /// Path to the directory with some control flags for server maintenance. + ConfigurationPtr config; /// Global configuration settings. + Databases databases; /// List of databases and tables in them. FormatFactory format_factory; /// Formats. mutable std::shared_ptr embedded_dictionaries; /// Metrica's dictionaeis. Have lazy initialization. @@ -483,6 +485,23 @@ void Context::setFlagsPath(const String & path) shared->flags_path = path; } +void Context::setConfig(const ConfigurationPtr & config) +{ + auto lock = getLock(); + shared->config = config; +} + +ConfigurationPtr Context::getConfig() const +{ + auto lock = getLock(); + return shared->config; +} + +Poco::Util::AbstractConfiguration & Context::getConfigRef() const +{ + auto lock = getLock(); + return shared->config ? *shared->config : Poco::Util::Application::instance().config(); +} void Context::setUsersConfig(const ConfigurationPtr & config) { @@ -498,7 +517,6 @@ ConfigurationPtr Context::getUsersConfig() return shared->users_config; } - void Context::calculateUserSettings() { auto lock = getLock(); @@ -1020,7 +1038,7 @@ const EmbeddedDictionaries & Context::getEmbeddedDictionariesImpl(const bool thr std::lock_guard lock(shared->embedded_dictionaries_mutex); if (!shared->embedded_dictionaries) - shared->embedded_dictionaries = std::make_shared(throw_on_error); + shared->embedded_dictionaries = std::make_shared(*this->global_context, throw_on_error); return *shared->embedded_dictionaries; } @@ -1207,7 +1225,10 @@ std::pair Context::getInterserverIOAddress() const UInt16 Context::getTCPPort() const { - return Poco::Util::Application::instance().config().getInt("tcp_port"); + auto lock = getLock(); + + auto & config = getConfigRef(); + return config.getInt("tcp_port"); } @@ -1234,7 +1255,7 @@ Clusters & Context::getClusters() const std::lock_guard lock(shared->clusters_mutex); if (!shared->clusters) { - auto & config = shared->clusters_config ? *shared->clusters_config : Poco::Util::Application::instance().config(); + auto & config = shared->clusters_config ? *shared->clusters_config : getConfigRef(); shared->clusters = std::make_unique(config, settings); } } @@ -1280,7 +1301,7 @@ QueryLog & Context::getQueryLog() if (!global_context) throw Exception("Logical error: no global context for query log", ErrorCodes::LOGICAL_ERROR); - auto & config = Poco::Util::Application::instance().config(); + auto & config = getConfigRef(); String database = config.getString("query_log.database", "system"); String table = config.getString("query_log.table", "query_log"); @@ -1299,7 +1320,7 @@ PartLog * Context::getPartLog(const String & database, const String & table) { auto lock = getLock(); - auto & config = Poco::Util::Application::instance().config(); + auto & config = getConfigRef(); if (!config.has("part_log")) return nullptr; @@ -1340,7 +1361,7 @@ CompressionMethod Context::chooseCompressionMethod(size_t part_size, double part if (!shared->compression_method_selector) { constexpr auto config_name = "compression"; - auto & config = Poco::Util::Application::instance().config(); + auto & config = getConfigRef(); if (config.has(config_name)) shared->compression_method_selector = std::make_unique(config, "compression"); @@ -1358,7 +1379,7 @@ const MergeTreeSettings & Context::getMergeTreeSettings() if (!shared->merge_tree_settings) { - auto & config = Poco::Util::Application::instance().config(); + auto & config = getConfigRef(); shared->merge_tree_settings = std::make_unique(); shared->merge_tree_settings->loadFromConfig("merge_tree", config); } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index f2e5855bc6b..93cc7531c92 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -128,12 +128,16 @@ public: using ConfigurationPtr = Poco::AutoPtr; + /// Global application configuration settings. + void setConfig(const ConfigurationPtr & config); + ConfigurationPtr getConfig() const; + Poco::Util::AbstractConfiguration & getConfigRef() const; + /** Take the list of users, quotas and configuration profiles from this config. * The list of users is completely replaced. * The accumulated quota values are not reset if the quota is not deleted. */ void setUsersConfig(const ConfigurationPtr & config); - ConfigurationPtr getUsersConfig(); /// Must be called before getClientInfo. diff --git a/dbms/src/Interpreters/EmbeddedDictionaries.cpp b/dbms/src/Interpreters/EmbeddedDictionaries.cpp index 4d1f11d0167..5a3471a554d 100644 --- a/dbms/src/Interpreters/EmbeddedDictionaries.cpp +++ b/dbms/src/Interpreters/EmbeddedDictionaries.cpp @@ -1,12 +1,15 @@ -#include -#include -#include #include -#include #include +#include +#include +#include + #include #include #include +#include + +#include namespace DB @@ -26,12 +29,14 @@ void EmbeddedDictionaries::handleException(const bool throw_on_error) const template bool EmbeddedDictionaries::reloadDictionary(MultiVersion & dictionary, const bool throw_on_error) { - if (Dictionary::isConfigured() && (!is_fast_start_stage || !dictionary.get())) + const auto & config = context.getConfigRef(); + + if (Dictionary::isConfigured(config) && (!is_fast_start_stage || !dictionary.get())) { try { auto new_dictionary = std::make_unique(); - new_dictionary->reload(); + new_dictionary->reload(config); dictionary.set(new_dictionary.release()); } catch (...) @@ -100,20 +105,16 @@ void EmbeddedDictionaries::reloadPeriodically() } -EmbeddedDictionaries::EmbeddedDictionaries(const bool throw_on_error, const int reload_period_) - : reload_period(reload_period_), log(&Logger::get("EmbeddedDictionaries")) +EmbeddedDictionaries::EmbeddedDictionaries(Context & context_, const bool throw_on_error) + : log(&Logger::get("EmbeddedDictionaries")) + , context(context_) + , reload_period(context_.getConfigRef().getInt("builtin_dictionaries_reload_interval", 3600)) { reloadImpl(throw_on_error); reloading_thread = std::thread([this] { reloadPeriodically(); }); } -EmbeddedDictionaries::EmbeddedDictionaries(const bool throw_on_error) - : EmbeddedDictionaries(throw_on_error, - Poco::Util::Application::instance().config().getInt("builtin_dictionaries_reload_interval", 3600)) -{} - - EmbeddedDictionaries::~EmbeddedDictionaries() { destroy.set(); diff --git a/dbms/src/Interpreters/EmbeddedDictionaries.h b/dbms/src/Interpreters/EmbeddedDictionaries.h index d71b4feb927..bb14d7391a1 100644 --- a/dbms/src/Interpreters/EmbeddedDictionaries.h +++ b/dbms/src/Interpreters/EmbeddedDictionaries.h @@ -23,6 +23,9 @@ class Context; class EmbeddedDictionaries { private: + Poco::Logger * log; + Context & context; + MultiVersion regions_hierarchies; MultiVersion tech_data_hierarchy; MultiVersion regions_names; @@ -35,8 +38,6 @@ private: std::thread reloading_thread; Poco::Event destroy; - Poco::Logger * log; - void handleException(const bool throw_on_error) const; @@ -54,9 +55,7 @@ private: public: /// Every reload_period seconds directories are updated inside a separate thread. - EmbeddedDictionaries(const bool throw_on_error, const int reload_period_); - - EmbeddedDictionaries(const bool throw_on_error); + EmbeddedDictionaries(Context & context, const bool throw_on_error); ~EmbeddedDictionaries(); diff --git a/dbms/src/Interpreters/ExternalDictionaries.cpp b/dbms/src/Interpreters/ExternalDictionaries.cpp index 4d3dc137131..3b6ffb84428 100644 --- a/dbms/src/Interpreters/ExternalDictionaries.cpp +++ b/dbms/src/Interpreters/ExternalDictionaries.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -95,7 +96,7 @@ std::set getDictionariesConfigPaths(const Poco::Util::AbstractConfi void ExternalDictionaries::reloadImpl(const bool throw_on_error) { - const auto config_paths = getDictionariesConfigPaths(Poco::Util::Application::instance().config()); + const auto config_paths = getDictionariesConfigPaths(context.getConfigRef()); for (const auto & config_path : config_paths) { diff --git a/dbms/src/Server/HTTPHandler.cpp b/dbms/src/Server/HTTPHandler.cpp index 57ba1491e3d..8a421b94ff2 100644 --- a/dbms/src/Server/HTTPHandler.cpp +++ b/dbms/src/Server/HTTPHandler.cpp @@ -130,9 +130,10 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti } -static std::chrono::steady_clock::duration parseSessionTimeout(const HTMLForm & params) +static std::chrono::steady_clock::duration parseSessionTimeout( + const Poco::Util::AbstractConfiguration & config, + const HTMLForm & params) { - const auto & config = Poco::Util::Application::instance().config(); unsigned session_timeout = config.getInt("default_session_timeout", 60); if (params.has("session_timeout")) @@ -245,7 +246,7 @@ void HTTPHandler::processQuery( if (session_is_set) { session_id = params.get("session_id"); - session_timeout = parseSessionTimeout(params); + session_timeout = parseSessionTimeout(server.config(), params); std::string session_check = params.get("session_check", ""); session = context.acquireSession(session_id, session_timeout, session_check == "1"); diff --git a/dbms/src/Server/MetricsTransmitter.cpp b/dbms/src/Server/MetricsTransmitter.cpp index 8b65fc72f1c..bf0ae0e4186 100644 --- a/dbms/src/Server/MetricsTransmitter.cpp +++ b/dbms/src/Server/MetricsTransmitter.cpp @@ -1,15 +1,21 @@ #include "MetricsTransmitter.h" -#include -#include -#include +#include +#include + #include #include #include -#include + +#include + +#include +#include + namespace DB { + MetricsTransmitter::~MetricsTransmitter() { try @@ -32,7 +38,7 @@ MetricsTransmitter::~MetricsTransmitter() void MetricsTransmitter::run() { - auto & config = Poco::Util::Application::instance().config(); + const auto & config = context.getConfigRef(); auto interval = config.getInt(config_name + ".interval", 60); const std::string thread_name = "MericsTrns " + std::to_string(interval) + "s"; @@ -63,7 +69,7 @@ void MetricsTransmitter::run() void MetricsTransmitter::transmit(std::vector & prev_counters) { - auto & config = Poco::Util::Application::instance().config(); + const auto & config = context.getConfigRef(); auto async_metrics_values = async_metrics.getValues(); GraphiteWriter::KeyValueVector key_vals{}; diff --git a/dbms/src/Server/MetricsTransmitter.h b/dbms/src/Server/MetricsTransmitter.h index 4bee6ad0c7d..e85113ad141 100644 --- a/dbms/src/Server/MetricsTransmitter.h +++ b/dbms/src/Server/MetricsTransmitter.h @@ -10,7 +10,10 @@ namespace DB { + class AsynchronousMetrics; +class Context; + /** Automatically sends * - difference of ProfileEvents; @@ -21,8 +24,12 @@ class AsynchronousMetrics; class MetricsTransmitter { public: - MetricsTransmitter(const AsynchronousMetrics & async_metrics, const std::string & config_name) - : async_metrics{async_metrics}, config_name{config_name} + MetricsTransmitter(Context & context_, + const AsynchronousMetrics & async_metrics_, + const std::string & config_name_) + : context(context_) + , async_metrics(async_metrics_) + , config_name(config_name_) { } ~MetricsTransmitter(); @@ -31,6 +38,8 @@ private: void run(); void transmit(std::vector & prev_counters); + Context & context; + const AsynchronousMetrics & async_metrics; const std::string config_name; @@ -43,4 +52,5 @@ private: static constexpr auto current_metrics_path_prefix = "ClickHouse.Metrics."; static constexpr auto asynchronous_metrics_path_prefix = "ClickHouse.AsynchronousMetrics."; }; + } diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index 4ad0b9ec719..e7820e296de 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -530,7 +530,8 @@ int Server::main(const std::vector & args) std::vector> metrics_transmitters; for (const auto & graphite_key : DB::getMultipleKeysFromConfig(config(), "", "graphite")) { - metrics_transmitters.emplace_back(std::make_unique(async_metrics, graphite_key)); + metrics_transmitters.emplace_back(std::make_unique( + *global_context, async_metrics, graphite_key)); } SessionCleaner session_cleaner(*global_context); diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp index adf212a9a59..b0240553ae5 100644 --- a/dbms/src/Storages/StorageFactory.cpp +++ b/dbms/src/Storages/StorageFactory.cpp @@ -195,7 +195,7 @@ static void appendGraphitePattern(const Context & context, static void setGraphitePatternsFromConfig(const Context & context, const String & config_element, Graphite::Params & params) { - const Poco::Util::AbstractConfiguration & config = Poco::Util::Application::instance().config(); + const auto & config = context.getConfigRef(); if (!config.has(config_element)) throw Exception("No '" + config_element + "' element in configuration file", diff --git a/dbms/src/Storages/System/StorageSystemGraphite.cpp b/dbms/src/Storages/System/StorageSystemGraphite.cpp index 016aeaa304d..109d6baafa7 100644 --- a/dbms/src/Storages/System/StorageSystemGraphite.cpp +++ b/dbms/src/Storages/System/StorageSystemGraphite.cpp @@ -1,11 +1,12 @@ #include -#include #include #include +#include #include #include #include +#include #include @@ -63,9 +64,10 @@ static Pattern readOnePattern( return pattern; } -static std::vector readPatterns(const std::string & section) +static std::vector readPatterns( + const AbstractConfiguration & config, + const std::string & section) { - const AbstractConfiguration & config = Application::instance().config(); AbstractConfiguration::Keys keys; std::vector result; size_t count = 0; @@ -92,9 +94,8 @@ static std::vector readPatterns(const std::string & section) return result; } -static Strings getAllGraphiteSections() +static Strings getAllGraphiteSections(const AbstractConfiguration & config) { - const AbstractConfiguration & config = Application::instance().config(); Strings result; AbstractConfiguration::Keys keys; @@ -180,10 +181,12 @@ BlockInputStreams StorageSystemGraphite::read( col_is_default.column = std::make_shared(); block.insert(col_is_default); - Strings sections = getAllGraphiteSections(); + const auto & config = context.getConfigRef(); + + Strings sections = getAllGraphiteSections(config); for (const auto & section : sections) { - const auto patterns = readPatterns(section); + const auto patterns = readPatterns(config, section); for (const auto & pattern : patterns) { for (const auto & ret : pattern.retentions) From 8d7937cc300d8cb92c36f5fc0ec9e94c25cec5cb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 29 Aug 2017 17:08:09 +0300 Subject: [PATCH 05/32] Fixed bad translation [#CLICKHOUSE-2]. --- dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index ee85110d30d..d3177c41875 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -145,7 +145,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::read( MergeTreeData::DataPartsVector parts = data.getDataPartsVector(); /// If query contains restrictions on the virtual column `_part` or `_part_index`, select only parts suitable for it. - /// The virtual column `_sample_factor - 1 / ` can be requested in the query. + /// The virtual column `_sample_factor` (which is equal to 1 / used sample rate) can be requested in the query. Names virt_column_names; Names real_column_names; From 221110454e5bc8c357928148a7d7378c396a1328 Mon Sep 17 00:00:00 2001 From: proller Date: Tue, 29 Aug 2017 20:13:20 +0300 Subject: [PATCH 06/32] Prefer mariadbclient --- debian/control | 2 +- libs/libmysqlxx/cmake/find_mysqlclient.cmake | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/control b/debian/control index a91d321d09d..2132f64a025 100644 --- a/debian/control +++ b/debian/control @@ -4,7 +4,7 @@ Maintainer: Alexey Milovidov Build-Depends: debhelper (>= 9), cmake, gcc-6, g++-6, - default-libmysqlclient-dev | libmysqlclient-dev, + libmariadbclient-dev | default-libmysqlclient-dev | libmysqlclient-dev, libicu-dev, libltdl-dev, libreadline-dev, diff --git a/libs/libmysqlxx/cmake/find_mysqlclient.cmake b/libs/libmysqlxx/cmake/find_mysqlclient.cmake index ae77349e444..1e6d6decbb6 100644 --- a/libs/libmysqlxx/cmake/find_mysqlclient.cmake +++ b/libs/libmysqlxx/cmake/find_mysqlclient.cmake @@ -22,9 +22,9 @@ if (ENABLE_MYSQL) find_path (MYSQL_INCLUDE_DIR NAMES mysql/mysql.h PATHS ${MYSQL_INCLUDE_PATHS}) if (USE_STATIC_LIBRARIES) - find_library (STATIC_MYSQLCLIENT_LIB libmysqlclient.a PATHS ${MYSQL_LIB_PATHS}) + find_library (STATIC_MYSQLCLIENT_LIB mariadbclient mysqlclient PATHS ${MYSQL_LIB_PATHS}) else () - find_library (MYSQLCLIENT_LIB mysqlclient PATHS ${MYSQL_LIB_PATHS}) + find_library (MYSQLCLIENT_LIB mariadbclient mysqlclient PATHS ${MYSQL_LIB_PATHS}) endif () if (MYSQL_INCLUDE_DIR AND (STATIC_MYSQLCLIENT_LIB OR MYSQLCLIENT_LIB)) From 84660029ecd22202049924126070cd8b4e279d1a Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Tue, 29 Aug 2017 14:58:33 +0300 Subject: [PATCH 07/32] Don't leave trash part nodes in ZK in case of deleting non-existing duplicated parts. [#CLICKHOUSE-3252] --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../Storages/StorageReplicatedMergeTree.cpp | 82 +++++++++++++++---- 2 files changed, 69 insertions(+), 15 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 78a7c3188f0..bd6de8d05c7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1304,7 +1304,7 @@ void MergeTreeData::replaceParts(const DataPartsVector & remove, const DataParts for (const DataPartPtr & part : remove) { - part->remove_time = clear_without_timeout ? 0 : time(0); + part->remove_time = clear_without_timeout ? 0 : time(nullptr); if (data_parts.erase(part)) removePartContributionToColumnSizes(part); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 8f2ab206ec8..f28b46c5bd9 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3790,11 +3790,26 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK(Logger * log_) if (!count) return; + /// Part names that were successfully deleted from filesystem and should be deleted from ZooKeeper + Strings part_names; + auto remove_from_zookeeper = [&] () + { + LOG_DEBUG(log, "Removed " << part_names.size() << " old parts from filesystem. Removing them from ZooKeeper."); + + try + { + removePartsFromZooKeeper(zookeeper, part_names); + } + catch (...) + { + LOG_ERROR(log, "There is a problem with deleting parts from ZooKeeper: " << getCurrentExceptionMessage(false)); + } + }; + try { LOG_DEBUG(log, "Removing " << parts.size() << " old parts from filesystem"); - Strings part_names; while (!parts.empty()) { MergeTreeData::DataPartPtr & part = parts.back(); @@ -3802,42 +3817,81 @@ void StorageReplicatedMergeTree::clearOldPartsAndRemoveFromZK(Logger * log_) part_names.emplace_back(part->name); parts.pop_back(); } - - LOG_DEBUG(log, "Removed " << part_names.size() << " old parts from filesystem. Removing them from ZooKeeper."); - - try - { - removePartsFromZooKeeper(zookeeper, part_names); - } - catch (const zkutil::KeeperException & e) - { - LOG_ERROR(log, "There is a problem with deleting parts from ZooKeeper: " << getCurrentExceptionMessage(false)); - } } catch (...) { tryLogCurrentException(__PRETTY_FUNCTION__); + + /// Finalize deletion of parts already deleted from filesystem, rollback remaining parts data.addOldParts(parts); + remove_from_zookeeper(); + throw; } + /// Finalize deletion + remove_from_zookeeper(); + LOG_DEBUG(log, "Removed " << count << " old parts"); } +static int32_t tryMultiWithRetries(zkutil::ZooKeeperPtr & zookeeper, zkutil::Ops & ops) noexcept +{ + int32_t code; + try + { + code = zookeeper->tryMultiWithRetries(ops); + } + catch (const zkutil::KeeperException & e) + { + code = e.code; + } + + return code; +} + + void StorageReplicatedMergeTree::removePartsFromZooKeeper(zkutil::ZooKeeperPtr & zookeeper, const Strings & part_names) { zkutil::Ops ops; + auto it_first_node_in_batch = part_names.cbegin(); for (auto it = part_names.cbegin(); it != part_names.cend(); ++it) { removePartFromZooKeeper(*it, ops); - if (ops.size() >= zkutil::MULTI_BATCH_SIZE || next(it) == part_names.cend()) + auto it_next = std::next(it); + if (ops.size() >= zkutil::MULTI_BATCH_SIZE || it_next == part_names.cend()) { /// It is Ok to use multi with retries to delete nodes, because new nodes with the same names cannot appear here - zookeeper->tryMultiWithRetries(ops); + auto code = tryMultiWithRetries(zookeeper, ops); ops.clear(); + + if (code == ZNONODE) + { + /// Fallback + LOG_DEBUG(log, "There are no some part nodes in ZooKeeper, will remove part nodes sequentially"); + + for (auto it_in_batch = it_first_node_in_batch; it_in_batch != it_next; ++it_in_batch) + { + zkutil::Ops cur_ops; + removePartFromZooKeeper(*it_in_batch, cur_ops); + auto cur_code = tryMultiWithRetries(zookeeper, cur_ops); + + if (cur_code == ZNONODE) + LOG_DEBUG(log, "There is no part " << *it_in_batch << " in ZooKeeper, it was only in filesystem"); + else if (cur_code != ZOK) + LOG_WARNING(log, "Cannot remove part " << *it_in_batch << " from ZooKeeper: " << ::zerror(cur_code)); + } + } + else if (code != ZOK) + { + LOG_WARNING(log, "There was a problem with deleting " << (it_next - it_first_node_in_batch) + << " nodes from ZooKeeper: " << ::zerror(code)); + } + + it_first_node_in_batch = it_next; } } } From 812096b0faf5b81bcba3907e7d0d3af85ccf54cb Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Tue, 29 Aug 2017 21:06:37 +0300 Subject: [PATCH 08/32] Updated changelog. [#CLICKHOUSE-3] --- CHANGELOG.md | 4 ++++ CHANGELOG_RU.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7eb95a73b0..8a7a389bac9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# ClickHouse release 1.1.54284 + +* This is bugfix release for previous 1.1.54282 release. It fixes ZooKeeper nodes leak in `parts/` directory. + # ClickHouse release 1.1.54282 This is a bugfix release. The following bugs were fixed: diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md index c6f6034c95c..6ac2620cbac 100644 --- a/CHANGELOG_RU.md +++ b/CHANGELOG_RU.md @@ -1,3 +1,7 @@ +# Релиз ClickHouse 1.1.54284 + +* Релиз содержит изменения к предыдущему релизу 1.1.54282, которые исправляют утечку записей о кусках в ZooKeeper + # Релиз ClickHouse 1.1.54282 Релиз содержит исправления к предыдущему релизу 1.1.54276: From 04af7a33491775c6be65af35e5062901abda491c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 29 Aug 2017 21:51:48 +0300 Subject: [PATCH 09/32] Fixed bad translation [#CLICKHOUSE-2]. --- dbms/src/Storages/Distributed/DirectoryMonitor.cpp | 6 ++---- dbms/src/Storages/MergeTree/PKCondition.h | 10 +++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index b531953cde2..03831ba8428 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -57,16 +57,14 @@ namespace if (!user_pw_end || !colon) throw Exception{ "Shard address '" + address + "' does not match to 'user[:password]@host:port#default_database' pattern", - ErrorCodes::INCORRECT_FILE_NAME - }; + ErrorCodes::INCORRECT_FILE_NAME}; const bool has_pw = colon < user_pw_end; const char * host_end = has_pw ? strchr(user_pw_end + 1, ':') : colon; if (!host_end) throw Exception{ "Shard address '" + address + "' does not contain port", - ErrorCodes::INCORRECT_FILE_NAME - }; + ErrorCodes::INCORRECT_FILE_NAME}; const char * has_db = strchr(address.data(), '#'); const char * port_end = has_db ? has_db : address_end; diff --git a/dbms/src/Storages/MergeTree/PKCondition.h b/dbms/src/Storages/MergeTree/PKCondition.h index 98c1d32cec0..ad4ea02b387 100644 --- a/dbms/src/Storages/MergeTree/PKCondition.h +++ b/dbms/src/Storages/MergeTree/PKCondition.h @@ -20,7 +20,7 @@ class IFunction; using FunctionPtr = std::shared_ptr; -/** Range with open or closed ends; Perhaps unlimited. +/** Range with open or closed ends; possibly unbounded. */ struct Range { @@ -31,12 +31,12 @@ private: public: Field left; /// the left border, if any Field right; /// the right border, if any - bool left_bounded = false; /// limited to the left - bool right_bounded = false; /// limited to the right + bool left_bounded = false; /// bounded at the left + bool right_bounded = false; /// bounded at the right bool left_included = false; /// includes the left border, if any bool right_included = false; /// includes the right border, if any - /// The whole set. + /// The whole unversum. Range() {} /// One point. @@ -148,7 +148,7 @@ public: /// r to the right of me. if (r.left_bounded && right_bounded - && (less(right, r.left) /// ...} {... + && (less(right, r.left) /// ...} {... || ((!right_included || !r.left_included) /// ...) [... or ...] (... && equals(r.left, right)))) return false; From e4eb6f0cfb47704d37c09631c099c40feec63962 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 29 Aug 2017 22:02:07 +0300 Subject: [PATCH 10/32] Fixed bad translation [#CLICKHOUSE-2]. --- dbms/src/Storages/MergeTree/PKCondition.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/Storages/MergeTree/PKCondition.h b/dbms/src/Storages/MergeTree/PKCondition.h index ad4ea02b387..5c48df5becc 100644 --- a/dbms/src/Storages/MergeTree/PKCondition.h +++ b/dbms/src/Storages/MergeTree/PKCondition.h @@ -193,15 +193,15 @@ public: /** Condition on the index. * * Consists of the conditions for the key belonging to all possible ranges or sets, - * as well as logical links AND/OR/NOT above these conditions. + * as well as logical operators AND/OR/NOT above these conditions. * * Constructs a reverse polish notation from these conditions - * and can calculate (interpret) its feasibility over key ranges. + * and can calculate (interpret) its satisfiability over key ranges. */ class PKCondition { public: - /// Does not include the SAMPLE section. all_columns - the set of all columns of the table. + /// Does not take into account the SAMPLE section. all_columns - the set of all columns of the table. PKCondition( const SelectQueryInfo & query_info, const Context & context, From d1da6c3cfc620bcc9110820d0edede1172d7d018 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 29 Aug 2017 22:07:54 +0300 Subject: [PATCH 11/32] Fixed bad translation [#CLICKHOUSE-2]. --- dbms/src/Core/Field.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Core/Field.h b/dbms/src/Core/Field.h index dd96612e827..affa2dcec96 100644 --- a/dbms/src/Core/Field.h +++ b/dbms/src/Core/Field.h @@ -37,8 +37,8 @@ STRONG_TYPEDEF(TupleBackend, Tuple); /// Array and Tuple are different types wit * is not generalized, * but somewhat more efficient, and simpler. * - * Used to represent a unit value of one of several types in the RAM. - * Warning! Preferably, instead of single values, store the pieces of the columns. See Column.h + * Used to represent a single value of one of several types in memory. + * Warning! Prefer to use chunks of columns instead of single values. See Column.h */ class Field { From 1e68d10ce00e9b0a6898044e23328de136e23032 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Tue, 29 Aug 2017 16:23:04 +0300 Subject: [PATCH 12/32] Add max_network_bandwidth_for_user limit setting. [#CLICKHOUSE-3139] --- dbms/src/Common/Throttler.h | 13 ++++++++++--- .../Interpreters/ClusterProxy/executeQuery.cpp | 18 +++++++++++++++--- dbms/src/Interpreters/Context.cpp | 5 +++++ dbms/src/Interpreters/Context.h | 1 + dbms/src/Interpreters/Limits.h | 2 ++ dbms/src/Interpreters/ProcessList.cpp | 12 ++++++++++-- dbms/src/Interpreters/ProcessList.h | 13 +++++++++++++ 7 files changed, 56 insertions(+), 8 deletions(-) diff --git a/dbms/src/Common/Throttler.h b/dbms/src/Common/Throttler.h index 0b242b25110..9104d39da92 100644 --- a/dbms/src/Common/Throttler.h +++ b/dbms/src/Common/Throttler.h @@ -27,10 +27,11 @@ namespace ErrorCodes class Throttler { public: - Throttler(size_t max_speed_, size_t limit_, const char * limit_exceeded_exception_message_) - : max_speed(max_speed_), limit(limit_), limit_exceeded_exception_message(limit_exceeded_exception_message_) {} + Throttler(size_t max_speed_, size_t limit_, const char * limit_exceeded_exception_message_, + const std::shared_ptr & parent = nullptr) + : max_speed(max_speed_), limit(limit_), limit_exceeded_exception_message(limit_exceeded_exception_message_), parent(parent) {} - void add(size_t amount) + void add(const size_t amount) { size_t new_count; UInt64 elapsed_ns = 0; @@ -70,6 +71,9 @@ public: nanosleep(&sleep_ts, nullptr); /// NOTE Returns early in case of a signal. This is considered normal. } } + + if (parent) + parent->add(amount); } private: @@ -79,6 +83,9 @@ private: const char * limit_exceeded_exception_message = nullptr; Stopwatch watch {CLOCK_MONOTONIC_COARSE}; std::mutex mutex; + + /// Used to implement a hierarchy of throttlers + std::shared_ptr parent; }; diff --git a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp index cf1c6e6db36..cc48d346f95 100644 --- a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -39,13 +40,24 @@ BlockInputStreams executeQuery( Context new_context(context); new_context.setSettings(new_settings); + ThrottlerPtr user_level_throttler; + if (settings.limits.max_network_bandwidth_for_user) + if (auto process_list_element = context.getProcessListElement()) + if (auto user_process_list = process_list_element->user_process_list) + user_level_throttler = user_process_list->user_throttler; + /// Network bandwidth limit, if needed. ThrottlerPtr throttler; if (settings.limits.max_network_bandwidth || settings.limits.max_network_bytes) + { throttler = std::make_shared( - settings.limits.max_network_bandwidth, - settings.limits.max_network_bytes, - "Limit for bytes to send or receive over network exceeded."); + settings.limits.max_network_bandwidth, + settings.limits.max_network_bytes, + "Limit for bytes to send or receive over network exceeded.", + user_level_throttler); + } + else if (settings.limits.max_network_bandwidth_for_user) + throttler = user_level_throttler; for (const auto & shard_info : cluster->getShardsInfo()) stream_factory.createForShard(shard_info, query, query_ast, new_context, throttler, res); diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 42a4b1bcba0..f894fe12563 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1094,6 +1094,11 @@ ProcessList::Element * Context::getProcessListElement() return process_list_elem; } +const ProcessList::Element * Context::getProcessListElement() const +{ + return process_list_elem; +} + void Context::setUncompressedCache(size_t max_size_in_bytes) { diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 93cc7531c92..69875a3d1ad 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -269,6 +269,7 @@ public: void setProcessListElement(ProcessListElement * elem); /// Can return nullptr if the query was not inserted into the ProcessList. ProcessListElement * getProcessListElement(); + const ProcessListElement * getProcessListElement() const; /// List all queries. ProcessList & getProcessList(); diff --git a/dbms/src/Interpreters/Limits.h b/dbms/src/Interpreters/Limits.h index 6851257dabe..0ae83f01dcf 100644 --- a/dbms/src/Interpreters/Limits.h +++ b/dbms/src/Interpreters/Limits.h @@ -98,6 +98,8 @@ struct Limits M(SettingUInt64, max_network_bandwidth, 0) \ /** The maximum number of bytes to receive or transmit over the network, as part of the query. */ \ M(SettingUInt64, max_network_bytes, 0) \ + /** The maximum speed of data exchange over the network for the user in bytes per second. 0 - not bounded. */ \ + M(SettingUInt64, max_network_bandwidth_for_user, 0) #define DECLARE(TYPE, NAME, DEFAULT) \ TYPE NAME {DEFAULT}; diff --git a/dbms/src/Interpreters/ProcessList.cpp b/dbms/src/Interpreters/ProcessList.cpp index e4283df201a..a3f3ad410d0 100644 --- a/dbms/src/Interpreters/ProcessList.cpp +++ b/dbms/src/Interpreters/ProcessList.cpp @@ -101,6 +101,14 @@ ProcessList::EntryPtr ProcessList::insert( total_memory_tracker.setDescription("(total)"); user_process_list.user_memory_tracker.setNext(&total_memory_tracker); } + + if (settings.limits.max_network_bandwidth_for_user && !user_process_list.user_throttler) + { + user_process_list.user_throttler = std::make_shared(settings.limits.max_network_bandwidth_for_user, 0, + "Network bandwidth limit for a user exceeded."); + } + + res->get().user_process_list = &user_process_list; } } @@ -124,13 +132,13 @@ ProcessListEntry::~ProcessListEntry() /// This removes the memory_tracker of one request. parent.cont.erase(it); - ProcessList::UserToQueries::iterator user_process_list = parent.user_to_queries.find(user); + auto user_process_list = parent.user_to_queries.find(user); if (user_process_list != parent.user_to_queries.end()) { /// In case the request is canceled, the data about it is deleted from the map at the time of cancellation, and not here. if (!is_cancelled && !query_id.empty()) { - ProcessListForUser::QueryToElement::iterator element = user_process_list->second.queries.find(query_id); + auto element = user_process_list->second.queries.find(query_id); if (element != user_process_list->second.queries.end()) user_process_list->second.queries.erase(element); } diff --git a/dbms/src/Interpreters/ProcessList.h b/dbms/src/Interpreters/ProcessList.h index da284667e95..d8e70bed116 100644 --- a/dbms/src/Interpreters/ProcessList.h +++ b/dbms/src/Interpreters/ProcessList.h @@ -14,6 +14,7 @@ #include #include #include +#include namespace CurrentMetrics @@ -29,6 +30,7 @@ using StoragePtr = std::shared_ptr; using Tables = std::map; struct Settings; class IAST; +struct ProcessListForUser; /** List of currently executing queries. @@ -76,6 +78,9 @@ struct ProcessListElement /// Temporary tables could be registered here. Modify under mutex. Tables temporary_tables; + /// Be careful using it. For example, queries field could be modified concurrently. + const ProcessListForUser * user_process_list = nullptr; + protected: mutable std::mutex query_streams_mutex; @@ -169,6 +174,9 @@ struct ProcessListForUser /// Limit and counter for memory of all simultaneously running queries of single user. MemoryTracker user_memory_tracker; + + /// Count net usage for all simultaneously running queries of single user. + ThrottlerPtr user_throttler; }; @@ -214,10 +222,15 @@ private: mutable std::mutex mutex; mutable Poco::Condition have_space; /// Number of currently running queries has become less than maximum. + /// List of queries Container cont; size_t cur_size; /// In C++03 or C++11 and old ABI, std::list::size is not O(1). size_t max_size; /// 0 means no limit. Otherwise, when limit exceeded, an exception is thrown. + + /// Stores per-user info: queries, statistics and limits UserToQueries user_to_queries; + + /// Stores info about queries grouped by their priority QueryPriorities priorities; /// Limit and counter for memory of all simultaneously running queries. From f31ca9276fc441a1f4fc2b3b3447015031b71471 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 29 Aug 2017 23:20:21 +0300 Subject: [PATCH 13/32] Update ProcessList.h --- dbms/src/Interpreters/ProcessList.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/ProcessList.h b/dbms/src/Interpreters/ProcessList.h index d8e70bed116..c853f4a06bc 100644 --- a/dbms/src/Interpreters/ProcessList.h +++ b/dbms/src/Interpreters/ProcessList.h @@ -175,7 +175,7 @@ struct ProcessListForUser /// Limit and counter for memory of all simultaneously running queries of single user. MemoryTracker user_memory_tracker; - /// Count net usage for all simultaneously running queries of single user. + /// Count network usage for all simultaneously running queries of single user. ThrottlerPtr user_throttler; }; From 91abe434611840162a2542fe0db81bd443d2afc9 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 29 Aug 2017 23:22:23 +0300 Subject: [PATCH 14/32] Update ProcessList.cpp --- dbms/src/Interpreters/ProcessList.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/ProcessList.cpp b/dbms/src/Interpreters/ProcessList.cpp index a3f3ad410d0..99b256e77e9 100644 --- a/dbms/src/Interpreters/ProcessList.cpp +++ b/dbms/src/Interpreters/ProcessList.cpp @@ -145,8 +145,9 @@ ProcessListEntry::~ProcessListEntry() /// This removes the memory_tracker from the user. At this time, the memory_tracker that references it does not live. - /// If there are no more queries for the user, then we delete the record. + /// If there are no more queries for the user, then we delete the entry. /// This also clears the MemoryTracker for the user, and a message about the memory consumption is output to the log. + /// This also clears network bandwidth Throttler, so it will not count periods of inactivity. /// Sometimes it is important to reset the MemoryTracker, because it may accumulate skew /// due to the fact that there are cases when memory can be allocated while processing the request, but released later. if (user_process_list->second.queries.empty()) From ad46d061a4a71d65ea147c5e3b86b2526e736fa9 Mon Sep 17 00:00:00 2001 From: proller Date: Wed, 30 Aug 2017 17:08:19 +0300 Subject: [PATCH 15/32] Improve debian building --- debian/.pbuilderrc | 38 ++++++++++++++++++++++++++++++-------- debian/control | 2 +- debian/rules | 31 +++++++++++++++++++------------ release | 7 +++++-- 4 files changed, 55 insertions(+), 23 deletions(-) diff --git a/debian/.pbuilderrc b/debian/.pbuilderrc index 70f3c33a1cb..92520d78791 100644 --- a/debian/.pbuilderrc +++ b/debian/.pbuilderrc @@ -1,11 +1,12 @@ # ubuntu: # sudo DIST=trusty pbuilder create --configfile debian/.pbuilderrc && DIST=trusty pdebuild --configfile debian/.pbuilderrc -# sudo DIST=xenial pbuilder create --configfile debian/.pbuilderrc && DIST=xenial pdebuild --configfile debian/.pbuilderrc +##sudo DIST=xenial pbuilder create --configfile debian/.pbuilderrc && DIST=xenial pdebuild --configfile debian/.pbuilderrc # sudo DIST=zesty pbuilder create --configfile debian/.pbuilderrc && DIST=zesty pdebuild --configfile debian/.pbuilderrc +# sudo DIST=artful pbuilder create --configfile debian/.pbuilderrc && DIST=artful pdebuild --configfile debian/.pbuilderrc # debian: # sudo DIST=experimental pbuilder create --configfile debian/.pbuilderrc && DIST=experimental pdebuild --configfile debian/.pbuilderrc # sudo DIST=testing pbuilder create --configfile debian/.pbuilderrc && DIST=testing pdebuild --configfile debian/.pbuilderrc -# sudo DIST=unstable pbuilder create --configfile debian/.pbuilderrc && DIST=unstable pdebuild --configfile debian/.pbuilderrc +##sudo DIST=unstable pbuilder create --configfile debian/.pbuilderrc && DIST=unstable pdebuild --configfile debian/.pbuilderrc # sudo DIST=stable pbuilder create --configfile debian/.pbuilderrc && DIST=stable pdebuild --configfile debian/.pbuilderrc # TODO: # sudo DIST=zesty ARCH=i386 pbuilder create --configfile debian/.pbuilderrc && DIST=zesty ARCH=i386 pdebuild --configfile debian/.pbuilderrc @@ -25,14 +26,14 @@ DEBIAN_SUITES=($UNSTABLE_CODENAME $TESTING_CODENAME $STABLE_CODENAME $STABLE_BAC "experimental" "unstable" "testing" "stable") # List of Ubuntu suites. Update these when needed. -UBUNTU_SUITES=("zesty" "xenial" "trusty") +UBUNTU_SUITES=("artful" "zesty" "xenial" "trusty") # Mirrors to use. Update these to your preferred mirror. -#DEBIAN_MIRROR="deb.debian.org" +DEBIAN_MIRROR="deb.debian.org" #UBUNTU_MIRROR="mirrors.kernel.org" UBUNTU_MIRROR="mirror.yandex.ru" -DEBIAN_MIRROR="mirror.yandex.ru" +#DEBIAN_MIRROR="mirror.yandex.ru" # Optionally use the changelog of a package to determine the suite to use if # none set. @@ -89,9 +90,15 @@ elif $(echo ${UBUNTU_SUITES[@]} | grep -q $DIST); then OSNAME=ubuntu MIRRORSITE="http://$UBUNTU_MIRROR/$OSNAME/" COMPONENTS="main restricted universe multiverse" - OTHERMIRROR+="deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/$OSNAME $DIST main" + + case "$DIST" in + "trusty" | "xenial" ) + OTHERMIRROR+="deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/$OSNAME $DIST main" + ALLOWUNTRUSTED=yes + ;; + esac + # deb http://apt.llvm.org/zesty/ llvm-toolchain-zesty-5.0 main - ALLOWUNTRUSTED=yes else echo "Unknown distribution: $DIST" exit 1 @@ -99,7 +106,22 @@ fi echo "using $NAME $OSNAME $DIST $ARCH $LOGNAME" -CCACHEDIR=/var/cache/pbuilder/ccache +case "$DIST" in + "trusty") + # ccache broken + ;; + *) + CCACHEDIR=/var/cache/pbuilder/ccache + ;; +esac + +case "$DIST" in + "trusty" | "artful" | "experimental" | "unstable" ) + export DEB_CC=gcc-7 + export DEB_CXX=g++-7 + ;; +esac + export CCACHE_PREFIX= export DEB_BUILD_OPTIONS=parallel=`nproc` diff --git a/debian/control b/debian/control index 2132f64a025..6eb44d4bda4 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,7 @@ Priority: optional Maintainer: Alexey Milovidov Build-Depends: debhelper (>= 9), cmake, - gcc-6, g++-6, + gcc-7 | gcc-6, g++-7 | g++-6, libmariadbclient-dev | default-libmysqlclient-dev | libmysqlclient-dev, libicu-dev, libltdl-dev, diff --git a/debian/rules b/debian/rules index ae7875f6b83..c74140a2c51 100755 --- a/debian/rules +++ b/debian/rules @@ -19,22 +19,28 @@ DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH) #endif #DEB_BUILD_OPTIONS+=parallel=$(THREADS_COUNT) -DEB_CC ?= gcc-6 -DEB_CXX ?= g++-6 -DEB_CLANG ?= $(shell which clang-6.0 || which clang-5.0 || which clang-4.0 || which clang || which clang-3.9 || which clang-3.8) -# CMAKE_FLAGS_ADD += -DINTERNAL_COMPILER_EXECUTABLE=$(basename $(DEB_CLANG)) # TODO: this is actual only if you will also change clang name in copy_clang_binaries.sh +CMAKE_FLAGS += -DENABLE_TESTS=0 -DEB_BUILD_GNU_TYPE := $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE) -DEB_HOST_GNU_TYPE := $(shell dpkg-architecture -qDEB_HOST_GNU_TYPE) +DEB_CLANG ?= $(shell which clang-6.0 || which clang-5.0 || which clang-4.0 || which clang || which clang-3.9 || which clang-3.8) +# CMAKE_FLAGS += -DINTERNAL_COMPILER_EXECUTABLE=$(basename $(DEB_CLANG)) # TODO: this is actual only if you will also change clang name in copy_clang_binaries.sh + +#DEB_CC ?= gcc-6 +#DEB_CXX ?= g++-6 + +ifdef DEB_CXX + DEB_BUILD_GNU_TYPE := $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE) + DEB_HOST_GNU_TYPE := $(shell dpkg-architecture -qDEB_HOST_GNU_TYPE) ifeq ($(DEB_BUILD_GNU_TYPE),$(DEB_HOST_GNU_TYPE)) -CC := $(DEB_CC) -CXX := $(DEB_CXX) + CC := $(DEB_CC) + CXX := $(DEB_CXX) else -CC := $(DEB_HOST_GNU_TYPE)-$(DEB_CC) -CXX := $(DEB_HOST_GNU_TYPE)-$(DEB_CXX) + CC := $(DEB_HOST_GNU_TYPE)-$(DEB_CC) + CXX := $(DEB_HOST_GNU_TYPE)-$(DEB_CXX) +endif endif -CMAKE_FLAGS ?= -DCMAKE_CXX_COMPILER=`which $(CXX)` -DCMAKE_C_COMPILER=`which $(CC)` -DENABLE_TESTS=0 $(CMAKE_FLAGS_ADD) +CMAKE_FLAGS += -DCMAKE_CXX_COMPILER=`which $(CXX)` -DCMAKE_C_COMPILER=`which $(CC)` + ifndef DH_VERBOSE CMAKE_FLAGS += -DCMAKE_VERBOSE_MAKEFILE=0 endif @@ -76,9 +82,10 @@ override_dh_install: # In case building clickhouse-server, adding to package binary of clang, ld and header files - for dynamic compilation. mkdir -p $(DESTDIR)/usr/share/clickhouse/bin $(DESTDIR)/usr/share/clickhouse/headers +ifeq ($(USE_INTERNAL_COMPILER),1) CLANG=$(DEB_CLANG) debian/copy_clang_binaries.sh $(DESTDIR)/usr/share/clickhouse/bin/ CLANG=$(DEB_CLANG) ./copy_headers.sh . $(DESTDIR)/usr/share/clickhouse/headers - +endif # fake metrika files when private dir is empty mkdir -p debian/tmp/etc/clickhouse-server/metrika diff --git a/release b/release index 9e9d200499f..255f973eb91 100755 --- a/release +++ b/release @@ -7,6 +7,9 @@ cd $CURDIR source "./release_lib.sh" +DEB_CC=gcc-6 +DEB_CXX=g++-6 + CONTROL=debian/control DEBUILD_NOSIGN_OPTIONS="-us -uc" @@ -58,7 +61,7 @@ if [ -z "$THREAD_COUNT" ] ; then THREAD_COUNT=`nproc || grep -c ^processor /proc/cpuinfo` fi -CMAKE_FLAGS_ADD+=" $LIBTCMALLOC_OPTS -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE" +CMAKE_FLAGS+=" $LIBTCMALLOC_OPTS -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE" REVISION+=$VERSION_POSTFIX echo -e "\nCurrent revision is $REVISION" @@ -66,4 +69,4 @@ echo -e "\nCurrent revision is $REVISION" gen_changelog "$REVISION" "" "$AUTHOR" "" # Build (only binary packages). -debuild -e PATH -e SSH_AUTH_SOCK -e DEB_BUILD_OPTIONS=parallel=$THREAD_COUNT -e DEB_CC -e DEB_CXX -e DEB_CLANG -e CMAKE_FLAGS_ADD="$CMAKE_FLAGS_ADD" -b ${DEBUILD_NOSIGN_OPTIONS} ${DEBUILD_NODEPS_OPTIONS} +debuild -e PATH -e SSH_AUTH_SOCK -e DEB_BUILD_OPTIONS=parallel=$THREAD_COUNT -e DEB_CC=$DEB_CC -e DEB_CXX=$DEB_CXX -e USE_INTERNAL_COMPILER=1 -e CMAKE_FLAGS="$CMAKE_FLAGS" -b ${DEBUILD_NOSIGN_OPTIONS} ${DEBUILD_NODEPS_OPTIONS} From 5916d823ef9ef92f769db9d636d43fa2d0746a61 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Wed, 30 Aug 2017 16:40:03 +0300 Subject: [PATCH 16/32] Fix deadlock in case of using Buffer table with CLEAR COLUMN IN PARTITION. [#CLICKHOUSE-3238] --- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index f28b46c5bd9..3b8a6338fed 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1419,9 +1419,10 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry & auto entry_part_info = MergeTreePartInfo::fromPartName(entry.new_part_name); - /// We don't change table structure, only data in some parts, disable reading from them + /// We don't change table structure, only data in some parts + /// To disable reading from these parts, we will sequentially acquire write lock for each part inside alterDataPart() + /// If we will lock the whole table here, a deadlock can occur. For example, if use use Buffer table (CLICKHOUSE-3238) auto lock_read_structure = lockStructure(false); - auto lock_write_data = lockDataForAlter(); auto zookeeper = getZooKeeper(); From 79d6775274dd847f84b4cf46835931bde33399ee Mon Sep 17 00:00:00 2001 From: proller Date: Wed, 30 Aug 2017 18:20:42 +0300 Subject: [PATCH 17/32] Better .pbuilderrc description --- debian/.pbuilderrc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/debian/.pbuilderrc b/debian/.pbuilderrc index 92520d78791..d640d54871e 100644 --- a/debian/.pbuilderrc +++ b/debian/.pbuilderrc @@ -1,16 +1,20 @@ # ubuntu: +# prepare: +# ln -s gutsy /usr/share/debootstrap/scripts/artful +# echo "3.0 (native)" > debian/source/format +# build ubuntu: # sudo DIST=trusty pbuilder create --configfile debian/.pbuilderrc && DIST=trusty pdebuild --configfile debian/.pbuilderrc -##sudo DIST=xenial pbuilder create --configfile debian/.pbuilderrc && DIST=xenial pdebuild --configfile debian/.pbuilderrc # sudo DIST=zesty pbuilder create --configfile debian/.pbuilderrc && DIST=zesty pdebuild --configfile debian/.pbuilderrc # sudo DIST=artful pbuilder create --configfile debian/.pbuilderrc && DIST=artful pdebuild --configfile debian/.pbuilderrc -# debian: +# build debian: # sudo DIST=experimental pbuilder create --configfile debian/.pbuilderrc && DIST=experimental pdebuild --configfile debian/.pbuilderrc # sudo DIST=testing pbuilder create --configfile debian/.pbuilderrc && DIST=testing pdebuild --configfile debian/.pbuilderrc -##sudo DIST=unstable pbuilder create --configfile debian/.pbuilderrc && DIST=unstable pdebuild --configfile debian/.pbuilderrc # sudo DIST=stable pbuilder create --configfile debian/.pbuilderrc && DIST=stable pdebuild --configfile debian/.pbuilderrc +# BROKEN: +# sudo DIST=xenial pbuilder create --configfile debian/.pbuilderrc && DIST=xenial pdebuild --configfile debian/.pbuilderrc +# sudo DIST=unstable pbuilder create --configfile debian/.pbuilderrc && DIST=unstable pdebuild --configfile debian/.pbuilderrc # TODO: # sudo DIST=zesty ARCH=i386 pbuilder create --configfile debian/.pbuilderrc && DIST=zesty ARCH=i386 pdebuild --configfile debian/.pbuilderrc -# sudo DIST=experimental ARCH=arm64 pbuilder create --configfile debian/.pbuilderrc && DIST=experimental ARCH=arm64 pdebuild --configfile debian/.pbuilderrc # from https://wiki.debian.org/PbuilderTricks : From 0fa2606535092d75a4565be4870a371536ea14ef Mon Sep 17 00:00:00 2001 From: proller Date: Wed, 30 Aug 2017 18:20:59 +0300 Subject: [PATCH 18/32] Release: Fix version increment --- release_lib.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/release_lib.sh b/release_lib.sh index e1729f4412e..72c73ddbd5d 100644 --- a/release_lib.sh +++ b/release_lib.sh @@ -1,3 +1,6 @@ + +set +e + function get_revision { BASEDIR=$(dirname "${BASH_SOURCE[0]}") grep "set(VERSION_REVISION" ${BASEDIR}/dbms/cmake/version.cmake | sed 's/^.*VERSION_REVISION \(.*\))$/\1/' @@ -25,7 +28,7 @@ function gen_revision_author { while [ $succeeded -eq 0 ] && [ $attempts -le $max_attempts ]; do attempts=$(($attempts + 1)) REVISION=$(($REVISION + 1)) - ( git_tag_grep=`git tag | grep "$VERSION_PREFIX$REVISION$VERSION_POSTFIX"` ) || true + git_tag_grep=`git tag | grep "$VERSION_PREFIX$REVISION$VERSION_POSTFIX"` if [ "$git_tag_grep" == "" ]; then succeeded=1 fi @@ -36,13 +39,13 @@ function gen_revision_author { fi auto_message="Auto version update to" - ( git_log_grep=`git log --oneline --max-count=1 | grep "$auto_message"` ) || true + git_log_grep=`git log --oneline --max-count=1 | grep "$auto_message"` if [ "$git_log_grep" == "" ]; then tag="$VERSION_PREFIX$REVISION$VERSION_POSTFIX" # First tag for correct git describe echo -e "\nTrying to create tag: $tag" - git tag -a "$tag" -m "$tag" || true + git tag -a "$tag" -m "$tag" git_describe=`git describe` sed -i -- "s/VERSION_REVISION .*)/VERSION_REVISION $REVISION)/g;s/VERSION_DESCRIBE .*)/VERSION_DESCRIBE $git_describe)/g" dbms/cmake/version.cmake From 478470e1f18cef0b8aa64faf2dd8672831e32006 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Wed, 30 Aug 2017 16:15:42 +0300 Subject: [PATCH 19/32] Avoid segfaults and many other problems related with modifying global context in DDLWorker. [#CLICKHOUSE-3260] --- dbms/src/Interpreters/DDLWorker.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/DDLWorker.cpp b/dbms/src/Interpreters/DDLWorker.cpp index 1035082a773..fee78388266 100644 --- a/dbms/src/Interpreters/DDLWorker.cpp +++ b/dbms/src/Interpreters/DDLWorker.cpp @@ -506,7 +506,8 @@ bool DDLWorker::tryExecuteQuery(const String & query, const DDLTask & task, Exec try { - executeQuery(istr, ostr, false, context, nullptr); + Context local_context(context); + executeQuery(istr, ostr, false, local_context, nullptr); } catch (...) { From 495daffa163411fdc3f748f515085fcfaf1a1cb1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 30 Aug 2017 21:13:32 +0300 Subject: [PATCH 20/32] Miscellaneous [#CLICKHOUSE-2]. --- .../AggregateFunctions/AggregateFunctionAvg.h | 2 +- .../AggregateFunctionMinMaxAny.h | 2 +- .../AggregateFunctionQuantile.h | 4 +- .../AggregateFunctionQuantileTiming.h | 4 +- .../AggregateFunctionStatistics.h | 4 +- .../AggregateFunctions/AggregateFunctionSum.h | 2 +- .../AggregateFunctionUniq.h | 18 +-- .../AggregateFunctionUniqUpTo.h | 2 +- dbms/src/AggregateFunctions/Helpers.h | 144 +++++++++--------- .../src/AggregateFunctions/ReservoirSampler.h | 2 +- dbms/src/Common/ConfigProcessor.cpp | 2 +- dbms/src/Common/ConfigProcessor.h | 2 +- dbms/src/Common/ExternalTable.h | 2 +- dbms/src/Common/HashTable/TwoLevelHashMap.h | 2 +- dbms/src/Common/LRUCache.h | 2 +- dbms/src/Common/tests/hash_table.cpp | 4 +- dbms/src/Common/tests/lru_cache.cpp | 2 +- .../src/Common/tests/parallel_aggregation.cpp | 2 +- dbms/src/Core/Types.h | 2 +- dbms/src/DataTypes/DataTypeTraits.h | 4 +- dbms/src/DataTypes/NumberTraits.h | 20 +-- dbms/src/IO/WriteBufferFromVector.h | 2 +- dbms/src/IO/tests/read_buffer_aio.cpp | 2 +- dbms/src/IO/tests/write_buffer_aio.cpp | 2 +- dbms/src/Interpreters/tests/hash_map.cpp | 12 +- .../Interpreters/tests/hash_map_string.cpp | 6 +- .../Interpreters/tests/two_level_hash_map.cpp | 6 +- dbms/src/Storages/StorageLog.cpp | 2 +- .../Storages/StorageReplicatedMergeTree.cpp | 2 +- 29 files changed, 131 insertions(+), 131 deletions(-) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionAvg.h b/dbms/src/AggregateFunctions/AggregateFunctionAvg.h index 9cdff36c4bc..a073b279583 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionAvg.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionAvg.h @@ -23,7 +23,7 @@ struct AggregateFunctionAvgData /// Calculates arithmetic mean of numbers. template -class AggregateFunctionAvg final : public IUnaryAggregateFunction::Type>, AggregateFunctionAvg > +class AggregateFunctionAvg final : public IUnaryAggregateFunction::Type>, AggregateFunctionAvg> { public: String getName() const override { return "avg"; } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h index c27c8dfd568..42b2760e1f7 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h @@ -680,7 +680,7 @@ struct AggregateFunctionAnyHeavyData : Data template -class AggregateFunctionsSingleValue final : public IUnaryAggregateFunction > +class AggregateFunctionsSingleValue final : public IUnaryAggregateFunction> { private: DataTypePtr type; diff --git a/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h b/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h index eb7ae69aca0..32f4650a55b 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionQuantile.h @@ -34,7 +34,7 @@ struct AggregateFunctionQuantileData */ template class AggregateFunctionQuantile final - : public IUnaryAggregateFunction, AggregateFunctionQuantile > + : public IUnaryAggregateFunction, AggregateFunctionQuantile> { private: using Sample = typename AggregateFunctionQuantileData::Sample; @@ -108,7 +108,7 @@ public: */ template class AggregateFunctionQuantiles final - : public IUnaryAggregateFunction, AggregateFunctionQuantiles > + : public IUnaryAggregateFunction, AggregateFunctionQuantiles> { private: using Sample = typename AggregateFunctionQuantileData::Sample; diff --git a/dbms/src/AggregateFunctions/AggregateFunctionQuantileTiming.h b/dbms/src/AggregateFunctions/AggregateFunctionQuantileTiming.h index 9f4bb9eeda2..d07c3a7fcca 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionQuantileTiming.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionQuantileTiming.h @@ -790,7 +790,7 @@ public: template -class AggregateFunctionQuantileTiming final : public IUnaryAggregateFunction > +class AggregateFunctionQuantileTiming final : public IUnaryAggregateFunction> { private: double level; @@ -910,7 +910,7 @@ public: * Returns an array of results. */ template -class AggregateFunctionQuantilesTiming final : public IUnaryAggregateFunction > +class AggregateFunctionQuantilesTiming final : public IUnaryAggregateFunction> { private: QuantileLevels levels; diff --git a/dbms/src/AggregateFunctions/AggregateFunctionStatistics.h b/dbms/src/AggregateFunctions/AggregateFunctionStatistics.h index 89c0bfa2027..4ab999dd904 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionStatistics.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionStatistics.h @@ -112,7 +112,7 @@ private: template class AggregateFunctionVariance final : public IUnaryAggregateFunction, - AggregateFunctionVariance > + AggregateFunctionVariance> { public: String getName() const override { return Op::name; } @@ -371,7 +371,7 @@ template, - AggregateFunctionCovariance > + AggregateFunctionCovariance> { public: String getName() const override { return Op::name; } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionSum.h b/dbms/src/AggregateFunctions/AggregateFunctionSum.h index 54f84688d6e..4bb6a711c4f 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionSum.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionSum.h @@ -21,7 +21,7 @@ struct AggregateFunctionSumData /// Counts the sum of the numbers. template -class AggregateFunctionSum final : public IUnaryAggregateFunction::Type>, AggregateFunctionSum > +class AggregateFunctionSum final : public IUnaryAggregateFunction::Type>, AggregateFunctionSum> { public: String getName() const override { return "sum"; } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionUniq.h b/dbms/src/AggregateFunctions/AggregateFunctionUniq.h index b5b954b62a9..9c36b12a2cc 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionUniq.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionUniq.h @@ -122,7 +122,7 @@ struct BaseUniqCombinedData using Key = UInt32; using Set = CombinedCardinalityEstimator< Key, - HashSet >, + HashSet>, 16, 14, 17, @@ -141,7 +141,7 @@ struct BaseUniqCombinedData using Key = UInt64; using Set = CombinedCardinalityEstimator< Key, - HashSet >, + HashSet>, 16, 14, 17, @@ -252,7 +252,7 @@ struct OneAdder; template struct OneAdder::value || - std::is_same >::value>::type> + std::is_same>::value>::type> { template static void addImpl(Data & data, const IColumn & column, size_t row_num, @@ -273,10 +273,10 @@ struct OneAdder struct OneAdder >::value || - std::is_same >::value || - std::is_same >::value || - std::is_same >::value>::type> + std::is_same>::value || + std::is_same>::value || + std::is_same>::value || + std::is_same>::value>::type> { template static void addImpl(Data & data, const IColumn & column, size_t row_num, @@ -297,7 +297,7 @@ struct OneAdder struct OneAdder >::value>::type> + std::is_same>::value>::type> { template static void addImpl(Data & data, const IColumn & column, size_t row_num, @@ -326,7 +326,7 @@ struct OneAdder -class AggregateFunctionUniq final : public IUnaryAggregateFunction > +class AggregateFunctionUniq final : public IUnaryAggregateFunction> { public: String getName() const override { return Data::getName(); } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionUniqUpTo.h b/dbms/src/AggregateFunctions/AggregateFunctionUniqUpTo.h index d289bbcdc1b..3c40ad98c51 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionUniqUpTo.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionUniqUpTo.h @@ -116,7 +116,7 @@ struct AggregateFunctionUniqUpToData : AggregateFunctionUniqUpToData -class AggregateFunctionUniqUpTo final : public IUnaryAggregateFunction, AggregateFunctionUniqUpTo > +class AggregateFunctionUniqUpTo final : public IUnaryAggregateFunction, AggregateFunctionUniqUpTo> { private: UInt8 threshold = 5; /// Default value if the parameter is not specified. diff --git a/dbms/src/AggregateFunctions/Helpers.h b/dbms/src/AggregateFunctions/Helpers.h index d57218888da..b4977377679 100644 --- a/dbms/src/AggregateFunctions/Helpers.h +++ b/dbms/src/AggregateFunctions/Helpers.h @@ -18,18 +18,18 @@ namespace DB template